diff --git a/.github/workflows/check-configuration-files.yml b/.github/workflows/check-configuration-files.yml index f280e326c2..561b7d0019 100644 --- a/.github/workflows/check-configuration-files.yml +++ b/.github/workflows/check-configuration-files.yml @@ -32,14 +32,14 @@ on: jobs: build: name: Check thingsboard.yml file - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.13 uses: actions/setup-python@v3 with: - python-version: "3.10.2" + python-version: "3.13.2" architecture: "x64" env: AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache diff --git a/application/pom.xml b/application/pom.xml index c6058408f8..6ce77646f8 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -124,6 +124,10 @@ org.thingsboard.common edge-api + + org.thingsboard.common + edqs + org.thingsboard dao diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index ddfac0d768..e9cbf8ef8e 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -63,4 +63,21 @@ $$; -- UPDATE SAVE TIME SERIES NODES END -ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; \ No newline at end of file +ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; + +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START + +UPDATE tenant_profile +SET profile_data = profile_data + || jsonb_build_object( + 'configuration', profile_data->'configuration' || jsonb_build_object( + 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, + 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, + 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, + 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, + 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint + ) + ) +WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL; + +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 50a777840c..2c22941c2f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -33,7 +33,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.NotificationCenter; -import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; @@ -109,6 +109,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -233,7 +234,7 @@ public class ActorSystemContext { @Autowired(required = false) @Getter - private RuleEngineDeviceStateManager deviceStateManager; + private DeviceStateManager deviceStateManager; @Autowired @Getter @@ -544,6 +545,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldStateService calculatedFieldStateService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldQueueService calculatedFieldQueueService; + @Lazy @Autowired(required = false) @Getter @@ -644,6 +650,10 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; + @Value("${actors.calculated_fields.calculation_timeout:5}") + @Getter + private long cfCalculationResultTimeout; + @Getter @Setter private TbActorSystem actorSystem; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f7fc204c0f..9ab019097b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -134,14 +134,20 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM public void process(CalculatedFieldEntityDeleteMsg msg) { log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { - MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); - states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); - ctx.stop(ctx.getSelf()); + if (states.isEmpty()) { + msg.getCallback().onSuccess(); + } else { + MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); + states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + ctx.stop(ctx.getSelf()); + } } else { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var state = states.remove(cfId); if (state != null) { cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } else { + msg.getCallback().onSuccess(); } } } @@ -242,7 +248,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM justRestored = true; } if (state.isSizeOk()) { - if (state.updateState(newArgValues) || justRestored) { + if (state.updateState(ctx, newArgValues) || justRestored) { cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); @@ -274,36 +280,34 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - boolean stateSizeOk; - if (ctx.isInitialized() && state.isReady()) { - try { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + boolean stateSizeChecked = false; + try { + if (ctx.isInitialized() && state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); - stateSizeOk = state.isSizeOk(); - if (stateSizeOk) { + stateSizeChecked = true; + if (state.isSizeOk()) { cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); } } - } catch (Exception e) { - throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); } - } else { - state.checkStateSize(ctxId, ctx.getMaxStateSize()); - stateSizeOk = state.isSizeOk(); - if (stateSizeOk) { - callback.onSuccess(); // State was updated but no calculation performed; + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); + } finally { + if (!stateSizeChecked) { + state.checkStateSize(ctxId, ctx.getMaxStateSize()); + } + if (state.isSizeOk()) { + cfStateService.persistState(ctxId, state, callback); + } else { + removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); } - } - if (stateSizeOk) { - cfStateService.persistState(ctxId, state, callback); - } else { - removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); } } - private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) { + private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException { // We remove the state, but remember that it is over-sized in a local map. cfStateService.removeState(ctxId, new TbCallback() { @Override @@ -316,6 +320,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM callback.onFailure(ex); } }); + throw ex; } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index d26d89626e..1418821081 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; @@ -51,8 +50,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -202,7 +199,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (fieldsCount > 0) { MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); var entityId = msg.getEntityId(); - oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), callback)); + oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); } else { callback.onSuccess(); @@ -309,7 +306,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!entityIds.isEmpty()) { //TODO: no need to do this if we cache all created actors and know which one belong to us; var multiCallback = new MultipleTbCallback(entityIds.size(), callback); - entityIds.forEach(id -> deleteCfForEntity(entityId, cfId, multiCallback)); + entityIds.forEach(id -> deleteCfForEntity(id, cfId, multiCallback)); } else { callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index d162b6a09a..ee6758df6a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -28,8 +28,9 @@ import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; -import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -725,7 +726,7 @@ public class DefaultTbContext implements TbContext { } @Override - public RuleEngineDeviceStateManager getDeviceStateManager() { + public DeviceStateManager getDeviceStateManager() { return mainCtx.getDeviceStateManager(); } @@ -902,6 +903,11 @@ public class DefaultTbContext implements TbContext { return mainCtx.getCalculatedFieldService(); } + @Override + public RuleEngineCalculatedFieldQueueService getCalculatedFieldQueueService() { + return mainCtx.getCalculatedFieldQueueService(); + } + @Override public boolean isExternalNodeForceAck() { return mainCtx.isExternalNodeForceAck(); diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index fe85cf3f87..f899d0f480 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -34,6 +34,8 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfCtx; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; @@ -216,11 +218,14 @@ public class CalculatedFieldController extends BaseController { @RequestBody JsonNode inputParams) { String expression = inputParams.get("expression").asText(); Map arguments = Objects.requireNonNullElse( - JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { + JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference<>() { }), Collections.emptyMap() ); - ArrayList argNames = new ArrayList<>(arguments.keySet()); + + ArrayList ctxAndArgNames = new ArrayList<>(arguments.size() + 1); + ctxAndArgNames.add("ctx"); + ctxAndArgNames.addAll(arguments.keySet()); String output = ""; String errorText = ""; @@ -234,12 +239,20 @@ public class CalculatedFieldController extends BaseController { getTenantId(), tbelInvokeService, expression, - argNames.toArray(String[]::new) + ctxAndArgNames.toArray(String[]::new) ); - Object[] args = argNames.stream() - .map(arguments::get) - .toArray(); + + Object[] args = new Object[ctxAndArgNames.size()]; + args[0] = new TbelCfCtx(arguments); + for (int i = 1; i < ctxAndArgNames.size(); i++) { + var arg = arguments.get(ctxAndArgNames.get(i)); + if (arg instanceof TbelCfSingleValueArg svArg) { + args[i] = svArg.getValue(); + } else { + args[i] = arg; + } + } JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); output = JacksonUtil.toString(json); @@ -260,7 +273,8 @@ public class CalculatedFieldController extends BaseController { EntityType entityType = referencedEntityId.getEntityType(); switch (entityType) { case TENANT, CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); - default -> throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); + default -> + throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); } } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 0387e8c24a..7fd0b12077 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -20,6 +20,8 @@ import io.swagger.v3.oas.annotations.media.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -27,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -38,6 +41,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.query.EntityQueryService; @@ -55,6 +60,10 @@ public class EntityQueryController extends BaseController { @Autowired private EntityQueryService entityQueryService; + @Autowired + private EdqsService edqsService; + @Autowired + private EdqsApiService edqsApiService; private static final int MAX_PAGE_SIZE = 100; @@ -133,4 +142,16 @@ public class EntityQueryController extends BaseController { return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope); } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @PostMapping("/edqs/system/request") + public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) { + edqsService.processSystemRequest(request); + } + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @GetMapping("/edqs/enabled") + public boolean isEdqsApiEnabled() { + return edqsApiService.isEnabled(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java index ff4de0ce0c..1a3bdce1f6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java @@ -18,6 +18,7 @@ package org.thingsboard.server.controller; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; @@ -160,7 +161,12 @@ public class TenantProfileController extends BaseController { " \"rpcTtlDays\": 0,\n" + " \"queueStatsTtlDays\": 0,\n" + " \"ruleEngineExceptionsTtlDays\": 0,\n" + - " \"warnThreshold\": 0\n" + + " \"warnThreshold\": 0,\n" + + " \"maxCalculatedFieldsPerEntity\": 5,\n" + + " \"maxArgumentsPerCF\": 10,\n" + + " \"maxDataPointsPerRollingArg\": 1000,\n" + + " \"maxStateSizeInKBytes\": 32,\n" + + " \"maxSingleValueArgumentSizeInKBytes\": 2" + " }\n" + " },\n" + " \"default\": false\n" + @@ -172,7 +178,7 @@ public class TenantProfileController extends BaseController { @RequestMapping(value = "/tenantProfile", method = RequestMethod.POST) @ResponseBody public TenantProfile saveTenantProfile(@Parameter(description = "A JSON value representing the tenant profile.") - @RequestBody TenantProfile tenantProfile) throws ThingsboardException { + @Valid @RequestBody TenantProfile tenantProfile) throws ThingsboardException { TenantProfile oldProfile; if (tenantProfile.getId() == null) { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT_PROFILE, Operation.CREATE); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java index b688993a8d..eb86220361 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -18,13 +18,14 @@ package org.thingsboard.server.service.cf; import com.google.common.util.concurrent.FutureCallback; import org.thingsboard.rule.engine.api.AttributesDeleteRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import java.util.List; -public interface CalculatedFieldQueueService { +public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQueueService { /** * Filter CFs based on the request entity. Push to the queue if any matching CF exist; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java index cb6d3f0d6b..f95227bc24 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -15,22 +15,29 @@ */ package org.thingsboard.server.service.cf; +import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import org.rocksdb.Options; import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.utils.TbRocksDb; +import org.thingsboard.server.edqs.util.TbRocksDb; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory'") public class CfRocksDb extends TbRocksDb { - public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { + public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) { super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); } + @PostConstruct + @Override + public void init() { + super.init(); + } + @PreDestroy @Override public void close() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 2282124bd5..64487d9b3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -117,7 +117,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); - ctx.init(); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index f69f5cd50b..27bc0120c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -125,7 +125,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP } return Futures.whenAllComplete(argFutures.values()).call(() -> { var result = createStateByType(ctx); - result.updateState(argFutures.entrySet().stream() + result.updateState(ctx, argFutures.entrySet().stream() .collect(Collectors.toMap( Entry::getKey, // Keep the key as is entry -> { @@ -273,7 +273,8 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); - int limit = argument.getLimit() == 0 ? (int) maxDataPoints : argument.getLimit(); + int argumentLimit = argument.getLimit(); + int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 3ca0b220ff..ba4be6ace6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -47,6 +47,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Set; @@ -89,6 +90,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback callback) { + pushRequestToQueue(request, null, callback); + } + @Override public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); @@ -97,6 +103,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } + @Override + public void pushRequestToQueue(AttributesSaveRequest request, FutureCallback callback) { + pushRequestToQueue(request, null, callback); + } + @Override public void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); @@ -156,15 +167,19 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + List entries = request.getEntries(); - List versions = result.getVersions(); + List versions = result != null ? result.getVersions() : Collections.emptyList(); + for (int i = 0; i < entries.size(); i++) { - long tsVersion = versions.get(i); - TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); - telemetryMsg.addTsData(tsProto); + TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); + if (result != null) { + tsProtoBuilder.setVersion(versions.get(i)); + } + telemetryMsg.addTsData(tsProtoBuilder.build()); } - msg.setTelemetryMsg(telemetryMsg.build()); + msg.setTelemetryMsg(telemetryMsg.build()); return msg.build(); } @@ -175,9 +190,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); List entries = request.getEntries(); for (int i = 0; i < entries.size(); i++) { - long attrVersion = versions.get(i); - AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); - telemetryMsg.addAttrData(attrProto); + AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder(); + if (versions != null) { + attrProtoBuilder.setVersion(versions.get(i)); + } + telemetryMsg.addAttrData(attrProtoBuilder.build()); } msg.setTelemetryMsg(telemetryMsg.build()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d59a58f296..80b003b3cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -25,6 +25,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; + @Data @AllArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -43,7 +45,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } @Override - public boolean updateState(Map argumentValues) { + public boolean updateState(CalculatedFieldCtx ctx, Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } @@ -53,6 +55,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { for (Map.Entry entry : argumentValues.entrySet()) { String key = entry.getKey(); ArgumentEntry newEntry = entry.getValue(); + + checkArgumentSize(key, newEntry, ctx); + ArgumentEntry existingEntry = arguments.get(key); if (existingEntry == null || newEntry.isForceResetPrevious()) { @@ -81,6 +86,18 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } } + @Override + public void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx) { + if (entry instanceof TsRollingArgumentEntry) { + return; + } + if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (ctx.getMaxSingleValueArgumentSize() > 0 && toSingleValueArgumentProto(name, singleValueArgumentEntry).getSerializedSize() > ctx.getMaxSingleValueArgumentSize()) { + throw new IllegalArgumentException("Single value size exceeds the maximum allowed limit. The argument will not be used for calculation."); + } + } + } + protected abstract void validateNewEntry(ArgumentEntry newEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index d3b7020c25..e5a3d0e05e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -71,6 +71,7 @@ public class CalculatedFieldCtx { private long maxDataPointsPerRollingArg; private long maxStateSize; + private long maxSingleValueArgumentSize; public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { this.calculatedField = calculatedField; @@ -104,6 +105,7 @@ public class CalculatedFieldCtx { this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; + this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; } public void init() { @@ -134,11 +136,14 @@ public class CalculatedFieldCtx { throw new IllegalArgumentException("TBEL script engine is disabled!"); } + List ctxAndArgNames = new ArrayList<>(argNames.size() + 1); + ctxAndArgNames.add("ctx"); + ctxAndArgNames.addAll(argNames); return new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, expression, - argNames.toArray(String[]::new) + ctxAndArgNames.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 70587d8b1e..fc4ba513d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,7 +44,7 @@ public interface CalculatedFieldState { void setRequiredArguments(List requiredArguments); - boolean updateState(Map argumentValues); + boolean updateState(CalculatedFieldCtx ctx, Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); @@ -60,4 +60,6 @@ public interface CalculatedFieldState { void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); + void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 00d9ccbe5a..bf00f1b0b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -23,11 +23,17 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfCtx; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; @Data @Slf4j @@ -49,11 +55,20 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Object[] args = ctx.getArgNames().stream() - .map(this::toTbelArgument) - .toArray(); - - ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args); + Map arguments = new LinkedHashMap<>(); + List args = new ArrayList<>(ctx.getArgNames().size() + 1); + args.add(new Object()); // first element is a ctx, but we will set it later; + for (String argName : ctx.getArgNames()) { + var arg = toTbelArgument(argName); + arguments.put(argName, arg); + if (arg instanceof TbelCfSingleValueArg svArg) { + args.add(svArg.getValue()); + } else { + args.add(arg); + } + } + args.set(0, new TbelCfCtx(arguments)); + ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray()); Output output = ctx.getOutput(); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 25008a60bd..480b334ac3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.BasicKvEntry; @@ -64,7 +65,19 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult)))); + Object result; + Integer decimals = output.getDecimalsByDefault(); + if (decimals != null) { + if (decimals.equals(0)) { + result = TbUtils.toInt(expressionResult); + } else { + result = TbUtils.toFixed(expressionResult, decimals); + } + } else { + result = expressionResult; + } + + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result)))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index a064a99935..bfe9eed24f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -42,13 +42,17 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); - this.version = entry.getVersion(); + if (entry.hasVersion()) { + this.version = entry.getVersion(); + } this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); } public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); - this.version = entry.getVersion(); + if (entry.hasVersion()) { + this.version = entry.getVersion(); + } this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); } @@ -100,10 +104,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); this.version = newVersion; - BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); - if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) { - return false; - } this.kvEntryValue = singleValueEntry.getKvEntryValue(); return true; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1114d993a3..b5a680a072 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -89,7 +89,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { for (var e : tsRecords.entrySet()) { values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); } - return new TbelCfTsRollingArg(limit, timeWindow, values); + return new TbelCfTsRollingArg(timeWindow, values); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 6f9845b9a5..c62a551310 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -82,6 +82,11 @@ public class EdgeEventSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + try { if (!isValidSaveEntityEventForEdgeProcessing(event)) { return; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index a371582409..6fc6d5bad9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -152,8 +152,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { entityData = new HashMap<>(); attributes = JacksonUtil.newObjectNode(); for (AttributeKvEntry attr : ssAttributes) { - if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(attr.getKey()) - && !DefaultDeviceStateService.INACTIVITY_TIMEOUT.equals(attr.getKey())) { + if (DefaultDeviceStateService.ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT.contains(attr.getKey())) { continue; } if (attr.getDataType() == DataType.BOOLEAN && attr.getBooleanValue().isPresent()) { @@ -200,7 +199,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { } Map> tsData = new HashMap<>(); for (TsKvEntry tsKvEntry : tsKvEntries) { - if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(tsKvEntry.getKey())) { + if (DefaultDeviceStateService.ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT.contains(tsKvEntry.getKey())) { continue; } tsData.computeIfAbsent(tsKvEntry.getTs(), k -> new HashMap<>()).put(tsKvEntry.getKey(), tsKvEntry.getValue()); diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java new file mode 100644 index 0000000000..51c963ed2f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; + +import java.util.UUID; + +@Service +@Slf4j +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')") +public class DefaultEdqsApiService implements EdqsApiService { + + private final EdqsPartitionService edqsPartitionService; + private final EdqsClientQueueFactory queueFactory; + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + + @Value("${queue.edqs.api.auto_enable:true}") + private boolean autoEnable; + + private Boolean apiEnabled = null; + + @PostConstruct + private void init() { + requestTemplate = queueFactory.createEdqsRequestTemplate(); + requestTemplate.init(); + } + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + var requestMsg = ToEdqsMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTs(System.currentTimeMillis()) + .setRequestMsg(TransportProtos.EdqsRequestMsg.newBuilder() + .setValue(JacksonUtil.toString(request)) + .build()); + if (customerId != null && !customerId.isNullUid()) { + requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits()); + requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits()); + } + + Integer partition = edqsPartitionService.resolvePartition(tenantId); + ListenableFuture> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition); + return Futures.transform(resultFuture, msg -> { + TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg(); + return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class); + }, MoreExecutors.directExecutor()); + } + + @Override + public boolean isEnabled() { + return Boolean.TRUE.equals(apiEnabled); + } + + @Override + public void setEnabled(boolean enabled) { + if (enabled) { + log.info("Enabling EDQS API"); + } else { + log.info("Disabling EDQS API"); + } + apiEnabled = enabled; + } + + @Override + public boolean isSupported() { + return true; + } + + @Override + public boolean isAutoEnable() { + return autoEnable; + } + + @PreDestroy + private void stop() { + requestTemplate.stop(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java new file mode 100644 index 0000000000..e823dee4e7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -0,0 +1,298 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.google.protobuf.ByteString; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.environment.DistributedLock; +import org.thingsboard.server.queue.environment.DistributedLockService; +import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") +public class DefaultEdqsService implements EdqsService { + + private final EdqsClientQueueFactory queueFactory; + private final EdqsConverter edqsConverter; + private final EdqsSyncService edqsSyncService; + private final EdqsApiService edqsApiService; + private final DistributedLockService distributedLockService; + private final AttributesService attributesService; + private final EdqsPartitionService edqsPartitionService; + private final TopicService topicService; + private final TbServiceInfoProvider serviceInfoProvider; + @Autowired @Lazy + private TbClusterService clusterService; + @Autowired @Lazy + private HashPartitionService hashPartitionService; + + private EdqsProducer eventsProducer; + private ExecutorService executor; + private DistributedLock syncLock; + + @PostConstruct + private void init() { + executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); + eventsProducer = EdqsProducer.builder() + .queue(EdqsQueue.EVENTS) + .partitionService(edqsPartitionService) + .topicService(topicService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) + .build(); + syncLock = distributedLockService.getLock("edqs_sync"); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onStartUp() { + if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) { + return; + } + executor.submit(() -> { + try { + EdqsSyncState syncState = getSyncState(); + if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { + if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { + processSystemRequest(ToCoreEdqsRequest.builder() + .syncRequest(new EdqsSyncRequest()) + .build()); + } + } else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) { + // only if topic/RocksDB is not empty and sync is finished + edqsApiService.setEnabled(true); + } + } catch (Throwable e) { + log.error("Failed to start EDQS service", e); + } + }); + } + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) { + log.info("Processing system request {}", request); + if (request.getSyncRequest() != null) { + saveSyncState(EdqsSyncStatus.REQUESTED); + } + broadcast(request.toInternalMsg()); + } + + @Override + public void processSystemMsg(ToCoreEdqsMsg msg) { + executor.submit(() -> { + log.info("Processing system msg {}", msg); + try { + if (msg.getApiEnabled() != null) { + edqsApiService.setEnabled(msg.getApiEnabled()); + } + + if (msg.getSyncRequest() != null) { + syncLock.lock(); + try { + EdqsSyncState syncState = getSyncState(); + if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) { + log.info("EDQS sync is already finished"); + return; + } + + saveSyncState(EdqsSyncStatus.STARTED); + edqsSyncService.sync(); + saveSyncState(EdqsSyncStatus.FINISHED); + + if (edqsApiService.isSupported()) + if (edqsApiService.isAutoEnable()) { + log.info("EDQS sync is finished, auto-enabling API"); + broadcast(ToCoreEdqsMsg.builder() + .apiEnabled(Boolean.TRUE) + .build()); + } else { + log.info("EDQS sync is finished, but leaving API disabled"); + } + } catch (Exception e) { + log.error("Failed to complete sync", e); + saveSyncState(EdqsSyncStatus.FAILED); + } finally { + syncLock.unlock(); + } + } + } catch (Throwable e) { + log.error("Failed to process msg {}", msg, e); + } + }); + } + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType); + return; + } + onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity)); + } + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.UPDATED, object); + } + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType); + return; + } + onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE)); + } + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.DELETED, object); + } + + protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) { + executor.submit(() -> { + try { + String key = object.key(); + Long version = object.version(); + EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder() + .setKey(key) + .setObjectType(objectType.name()) + .setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object))) + .setEventType(eventType.name()); + if (version != null) { + eventMsg.setVersion(version); + } + eventsProducer.send(tenantId, objectType, key, ToEdqsMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTs(System.currentTimeMillis()) + .setEventMsg(eventMsg) + .build()); + } catch (Throwable e) { + log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e); + } + }); + } + + private boolean isEdqsType(TenantId tenantId, ObjectType objectType) { + if (objectType == null) { + return false; + } + if (!tenantId.isSysTenantId()) { + return ObjectType.edqsTypes.contains(objectType); + } else { + return ObjectType.edqsSystemTypes.contains(objectType); + } + } + + private void broadcast(ToCoreEdqsMsg msg) { + clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder() + .setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder() + .setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg)))) + .build()); + } + + @SneakyThrows + private EdqsSyncState getSyncState() { + EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS) + .flatMap(KvEntry::getJsonValue) + .map(value -> JacksonUtil.fromString(value, EdqsSyncState.class)) + .orElse(null); + log.info("EDQS sync state: {}", state); + return state; + } + + @SneakyThrows + private void saveSyncState(EdqsSyncStatus status) { + EdqsSyncState state = new EdqsSyncState(status); + log.info("New EDQS sync state: {}", state); + attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry( + new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)), + System.currentTimeMillis())).get(30, TimeUnit.SECONDS); + } + + @PreDestroy + private void stop() { + executor.shutdown(); + eventsProducer.stop(); + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + private static class EdqsSyncState { + private EdqsSyncStatus status; + } + + private enum EdqsSyncStatus { + REQUESTED, + STARTED, + FINISHED, + FAILED + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java new file mode 100644 index 0000000000..d77df5ced8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") +public class EdqsListener { + + private final EdqsService edqsService; + + @TransactionalEventListener(fallbackExecution = true) + public void onUpdate(SaveEntityEvent event) { + if (event.getEntityId() == null || event.getEntity() == null) { + return; + } + edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void onDelete(DeleteEntityEvent event) { + if (event.getEntityId() == null) { + return; + } + edqsService.onDelete(event.getTenantId(), event.getEntityId()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java new file mode 100644 index 0000000000..79e0e60983 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -0,0 +1,284 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sql.RelationEntity; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.sql.relation.RelationRepository; +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; + +@Slf4j +public abstract class EdqsSyncService { + + @Value("${queue.edqs.sync.entity_batch_size:10000}") + private int entityBatchSize; + @Value("${queue.edqs.sync.ts_batch_size:10000}") + private int tsBatchSize; + @Autowired + private EntityDaoRegistry entityDaoRegistry; + @Autowired + private AttributesDao attributesDao; + @Autowired + private KeyDictionaryDao keyDictionaryDao; + @Autowired + private RelationRepository relationRepository; + @Autowired + private TsKvLatestRepository tsKvLatestRepository; + @Autowired + @Lazy + private DefaultEdqsService edqsService; + + private final ConcurrentHashMap entityInfoMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap keys = new ConcurrentHashMap<>(); + + private final Map counters = new ConcurrentHashMap<>(); + + public abstract boolean isSyncNeeded(); + + public void sync() { + log.info("Synchronizing data to EDQS"); + long startTs = System.currentTimeMillis(); + counters.clear(); + + syncTenantEntities(); + syncRelations(); + loadKeyDictionary(); + syncAttributes(); + syncLatestTimeseries(); + + counters.clear(); + log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); + } + + private void process(TenantId tenantId, ObjectType type, EdqsObject object) { + AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger()); + if (counter.incrementAndGet() % 10000 == 0) { + log.info("Processed {} {} objects", counter.get(), type); + } + edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); + } + + private void syncTenantEntities() { + for (ObjectType type : edqsTenantTypes) { + log.info("Synchronizing {} entities to EDQS", type); + long ts = System.currentTimeMillis(); + EntityType entityType = type.toEntityType(); + Dao dao = entityDaoRegistry.getDao(entityType); + UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + while (true) { + var batch = dao.findNextBatch(lastId, entityBatchSize); + if (batch.isEmpty()) { + break; + } + for (EntityFields entityFields : batch) { + TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); + process(tenantId, type, new Entity(entityType, entityFields)); + } + EntityFields lastRecord = batch.get(batch.size() - 1); + lastId = lastRecord.getId(); + } + log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); + } + } + + private void syncRelations() { + log.info("Synchronizing relations to EDQS"); + long ts = System.currentTimeMillis(); + UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastFromEntityType = ""; + String lastRelationTypeGroup = ""; + String lastRelationType = ""; + UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastToEntityType = ""; + + while (true) { + List batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, + lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize); + if (batch.isEmpty()) { + break; + } + processRelationBatch(batch); + + RelationEntity lastRecord = batch.get(batch.size() - 1); + lastFromEntityId = lastRecord.getFromId(); + lastFromEntityType = lastRecord.getFromType(); + lastRelationTypeGroup = lastRecord.getRelationTypeGroup(); + lastRelationType = lastRecord.getRelationType(); + lastToEntityId = lastRecord.getToId(); + lastToEntityType = lastRecord.getToType(); + } + log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processRelationBatch(List relations) { + for (RelationEntity relation : relations) { + if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) { + EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); + if (entityIdInfo != null) { + process(entityIdInfo.tenantId(), RELATION, relation.toData()); + } else { + log.info("Relation from id not found: {} ", relation); + } + } + } + } + + private void loadKeyDictionary() { + log.info("Loading key dictionary"); + long ts = System.currentTimeMillis(); + var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000); + for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) { + keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey()); + } + log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncAttributes() { + log.info("Synchronizing attributes to EDQS"); + long ts = System.currentTimeMillis(); + + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastAttributeType = Integer.MIN_VALUE; + int lastAttributeKey = Integer.MIN_VALUE; + + while (true) { + List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize); + if (batch.isEmpty()) { + break; + } + processAttributeBatch(batch); + + AttributeKvEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getId().getEntityId(); + lastAttributeType = lastRecord.getId().getAttributeType(); + lastAttributeKey = lastRecord.getId().getAttributeKey(); + } + log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processAttributeBatch(List batch) { + for (AttributeKvEntity attribute : batch) { + attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); + UUID entityId = attribute.getId().getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); + if (entityIdInfo == null) { + log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId); + continue; + } + AttributeKv attributeKv = new AttributeKv( + EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId), + AttributeScope.valueOf(attribute.getId().getAttributeType()), + attribute.toData(), + attribute.getVersion()); + process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); + } + } + + private void syncLatestTimeseries() { + log.info("Synchronizing latest timeseries to EDQS"); + long ts = System.currentTimeMillis(); + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastKey = Integer.MIN_VALUE; + + while (true) { + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize); + if (batch.isEmpty()) { + break; + } + processTsKvLatestBatch(batch); + + TsKvLatestEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getEntityId(); + lastKey = lastRecord.getKey(); + } + log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processTsKvLatestBatch(List tsKvLatestEntities) { + for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { + try { + String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); + if (strKey == null) { + log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey()); + continue; + } + tsKvLatestEntity.setStrKey(strKey); + UUID entityUuid = tsKvLatestEntity.getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid); + if (entityIdInfo != null) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid); + LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion()); + process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv); + } + } catch (Exception e) { + log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); + } + } + } + + private String getStrKeyOrFetchFromDb(int key) { + String strKey = keys.get(key); + if (strKey != null) { + return strKey; + } else { + strKey = keyDictionaryDao.getKey(key); + if (strKey != null) { + keys.put(key, strKey); + } + } + return strKey; + } + + public record EntityIdInfo(EntityType entityType, TenantId tenantId) { + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java new file mode 100644 index 0000000000..4ef552521b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; + +import java.util.Collections; + +@Service +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") +public class KafkaEdqsSyncService extends EdqsSyncService { + + private final boolean syncNeeded; + + public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) { + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); + this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); + } + + @Override + public boolean isSyncNeeded() { + return syncNeeded; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java new file mode 100644 index 0000000000..904391f172 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") +public class LocalEdqsSyncService extends EdqsSyncService { + + private final EdqsRocksDb db; + + @Override + public boolean isSyncNeeded() { + return db.isNew(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 03bd7b4e66..648e89adc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -77,6 +77,11 @@ public class EntityStateSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); if (entityId == null) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 7219179b2c..50989ea5a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; @@ -33,6 +32,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.JavaSerDesUtil; import org.thingsboard.server.common.data.alarm.AlarmInfo; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -76,6 +77,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -88,7 +90,6 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.resource.TbImageService; @@ -146,9 +147,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; private QueueConsumerManager> firmwareStatesConsumer; @@ -175,7 +177,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) - .config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -251,7 +255,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, CoreQueueConfig config) throws Exception { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig config) throws Exception { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); @@ -289,6 +293,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> stateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, deviceInactivityTimeoutUpdateMsg.getInactivityTimeout())); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process device inactivity timeout update message for device [{}]", tenantId.getId(), deviceId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); @@ -723,10 +748,4 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, EdgeQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, StatsFactory statsFactory, EdgeContextComponent edgeCtx) { @@ -100,9 +100,9 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService, EdgeQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME)) - .config(EdgeQueueConfig.of(consumerPerPartition, pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -128,7 +128,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, EdgeQueueConfig edgeQueueConfig) throws InterruptedException { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig edgeQueueConfig) throws InterruptedException { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); @@ -285,10 +285,4 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService { + event.getNewPartitions().forEach((queueKey, partitions) -> { if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) { return; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 728f18bfff..46a42284b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -40,6 +40,7 @@ public class TbCoreConsumerStats { public static final String DEVICE_ACTIVITIES = "deviceActivity"; public static final String DEVICE_DISCONNECTS = "deviceDisconnect"; public static final String DEVICE_INACTIVITIES = "deviceInactivity"; + public static final String DEVICE_INACTIVITY_TIMEOUT_UPDATES = "deviceInactivityTimeoutUpdate"; public static final String TO_CORE_NF_OTHER = "coreNfOther"; // normally, there is no messages when codebase is fine public static final String TO_CORE_NF_COMPONENT_LIFECYCLE = "coreNfCompLfcl"; @@ -65,6 +66,7 @@ public class TbCoreConsumerStats { private final StatsCounter deviceActivitiesCounter; private final StatsCounter deviceDisconnectsCounter; private final StatsCounter deviceInactivitiesCounter; + private final StatsCounter deviceInactivityTimeoutUpdatesCounter; private final StatsCounter toCoreNfOtherCounter; private final StatsCounter toCoreNfComponentLifecycleCounter; @@ -95,6 +97,7 @@ public class TbCoreConsumerStats { this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES)); this.deviceDisconnectsCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_DISCONNECTS)); this.deviceInactivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITIES)); + this.deviceInactivityTimeoutUpdatesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITY_TIMEOUT_UPDATES)); // Core notification counters this.toCoreNfOtherCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NF_OTHER)); @@ -163,6 +166,11 @@ public class TbCoreConsumerStats { deviceInactivitiesCounter.increment(); } + public void log(TransportProtos.DeviceInactivityTimeoutUpdateProto msg) { + totalCounter.increment(); + deviceInactivityTimeoutUpdatesCounter.increment(); + } + public void log(TransportProtos.SubscriptionMgrMsgProto msg) { totalCounter.increment(); subscriptionMsgCounter.increment(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 31813803c3..c7f7f600a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -72,7 +72,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager deviceStateService; + private final TbClusterService clusterService; + + @Override + public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device connect event to local service. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime); + deviceStateService.onDeviceConnect(tenantId, deviceId, connectTime); + }, + () -> { + log.debug("[{}][{}] Sending device connect message to core. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime); + var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastConnectTime(connectTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceConnectMsg(deviceConnectMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device activity event to local service. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime); + deviceStateService.onDeviceActivity(tenantId, deviceId, activityTime); + }, + () -> { + log.debug("[{}][{}] Sending device activity message to core. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime); + var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastActivityTime(activityTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceActivityMsg(deviceActivityMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device disconnect event to local service. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime); + deviceStateService.onDeviceDisconnect(tenantId, deviceId, disconnectTime); + }, + () -> { + log.debug("[{}][{}] Sending device disconnect message to core. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime); + var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastDisconnectTime(disconnectTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceDisconnectMsg(deviceDisconnectMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device inactivity event to local service. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime); + deviceStateService.onDeviceInactivity(tenantId, deviceId, inactivityTime); + }, + () -> { + log.debug("[{}][{}] Sending device inactivity message to core. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime); + var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastInactivityTime(inactivityTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceInactivityMsg(deviceInactivityMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceInactivityTimeoutUpdate(TenantId tenantId, DeviceId deviceId, long inactivityTimeout, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device inactivity timeout update to local service. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout); + deviceStateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, inactivityTimeout); + }, + () -> { + log.debug("[{}][{}] Sending device inactivity timeout update message to core. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout); + var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(inactivityTimeout) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg) + .build(); + }, callback); + } + + private void forwardToDeviceStateService( + TenantId tenantId, DeviceId deviceId, + Consumer toDeviceStateService, + Supplier toCore, + TbCallback callback + ) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) { + try { + toDeviceStateService.accept(deviceStateService.get()); + } catch (Exception e) { + log.error("[{}][{}] Failed to process device connectivity event.", tenantId.getId(), deviceId.getId(), e); + callback.onFailure(e); + return; + } + callback.onSuccess(); + } else { + TransportProtos.ToCoreMsg toCoreMsg = toCore.get(); + clusterService.pushMsgToCore(tpi, deviceId.getId(), toCoreMsg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure)); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 11819ce5d8..cc476d377d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -96,6 +96,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -129,11 +130,10 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService PERSISTENT_TELEMETRY_KEYS = Arrays.asList( new EntityKey(EntityKeyType.TIME_SERIES, LAST_ACTIVITY_TIME), new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_ALARM_TIME), - new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_TIMEOUT), new EntityKey(EntityKeyType.TIME_SERIES, ACTIVITY_STATE), new EntityKey(EntityKeyType.TIME_SERIES, LAST_CONNECT_TIME), new EntityKey(EntityKeyType.TIME_SERIES, LAST_DISCONNECT_TIME), - new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT)); + new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT)); // inactivity timeout is always a server attribute, even when activity data is stored as time series private static final List PERSISTENT_ATTRIBUTE_KEYS = Arrays.asList( new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_ACTIVITY_TIME), @@ -143,8 +143,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, - LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); + public static final Set ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT = Set.of( + ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME + ); + + public static final Set ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT = Set.of( + ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT + ); + private static final List PERSISTENT_ENTITY_FIELDS = Arrays.asList( new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "type"), @@ -251,7 +257,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService= deviceState.getLastActivityTime()) { deviceState.setLastInactivityAlarmTime(0L); - save(deviceId, INACTIVITY_ALARM_TIME, 0L); + save(state.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0L); } } } @@ -583,7 +589,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService fetchDeviceState(Device device) { ListenableFuture future; if (persistToTelemetry) { - ListenableFuture> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES); - future = Futures.transform(tsData, extractDeviceStateData(device), MoreExecutors.directExecutor()); + ListenableFuture> timeseriesActivityDataFuture = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT); + ListenableFuture> inactivityTimeoutAttributeFuture = attributesService.find( + TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT + ); + + ListenableFuture> fullActivityDataFuture = Futures.whenAllSucceed(timeseriesActivityDataFuture, inactivityTimeoutAttributeFuture).call(() -> { + List activityTimeseries = Futures.getDone(timeseriesActivityDataFuture); + Optional inactivityTimeoutAttribute = Futures.getDone(inactivityTimeoutAttributeFuture); + + if (inactivityTimeoutAttribute.isPresent()) { + List result = new ArrayList<>(activityTimeseries.size() + 1); + result.addAll(activityTimeseries); + result.add(inactivityTimeoutAttribute.get()); + return result; + } else { + return activityTimeseries; + } + }, deviceStateCallbackExecutor); + + future = Futures.transform(fullActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor()); } else { - ListenableFuture> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, PERSISTENT_ATTRIBUTES); - future = Futures.transform(attrData, extractDeviceStateData(device), MoreExecutors.directExecutor()); + ListenableFuture> attributesActivityDataFuture = attributesService.find( + TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT + ); + future = Futures.transform(attributesActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor()); } - return transformInactivityTimeout(future); + return future; } - private ListenableFuture transformInactivityTimeout(ListenableFuture future) { - return Futures.transformAsync(future, deviceStateData -> { - if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != defaultInactivityTimeoutMs) { - return future; //fail fast - } - var attributesFuture = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT); - return Futures.transform(attributesFuture, attributes -> { - attributes.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> { - if (inactivityTimeout > 0) { - deviceStateData.getState().setInactivityTimeout(inactivityTimeout); - } - }); - return deviceStateData; - }, MoreExecutors.directExecutor()); - }, deviceStateCallbackExecutor); - } - - private Function, DeviceStateData> extractDeviceStateData(Device device) { + private Function, DeviceStateData> extractDeviceStateData(Device device) { return new Function<>() { @Nonnull @Override - public DeviceStateData apply(@Nullable List data) { + public DeviceStateData apply(@Nullable List data) { try { long lastActivityTime = getEntryValue(data, LAST_ACTIVITY_TIME, 0L); long inactivityAlarmTime = getEntryValue(data, INACTIVITY_ALARM_TIME, 0L); @@ -690,7 +698,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService 0 ? inactivityTimeout : defaultInactivityTimeoutMs) .build(); TbMsgMetaData md = new TbMsgMetaData(); md.putValue("deviceName", device.getName()); @@ -761,12 +769,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService deviceStateService; - private final TbClusterService clusterService; - - public DefaultRuleEngineDeviceStateManager( - TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService, - Optional deviceStateServiceOptional, TbClusterService clusterService - ) { - this.serviceInfoProvider = serviceInfoProvider; - this.partitionService = partitionService; - this.deviceStateService = deviceStateServiceOptional; - this.clusterService = clusterService; - } - - @Getter - private abstract static class ConnectivityEventInfo { - - private final TenantId tenantId; - private final DeviceId deviceId; - private final long eventTime; - - private ConnectivityEventInfo(TenantId tenantId, DeviceId deviceId, long eventTime) { - this.tenantId = tenantId; - this.deviceId = deviceId; - this.eventTime = eventTime; - } - - abstract void forwardToLocalService(); - - abstract TransportProtos.ToCoreMsg toQueueMsg(); - - } - - @Override - public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, connectTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceConnect(tenantId, deviceId, connectTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastConnectTime(connectTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceConnectMsg(deviceConnectMsg) - .build(); - } - }, callback); - } - - @Override - public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, activityTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceActivity(tenantId, deviceId, activityTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastActivityTime(activityTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceActivityMsg(deviceActivityMsg) - .build(); - } - }, callback); - } - - @Override - public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, disconnectTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceDisconnect(tenantId, deviceId, disconnectTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastDisconnectTime(disconnectTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceDisconnectMsg(deviceDisconnectMsg) - .build(); - } - }, callback); - } - - @Override - public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, inactivityTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceInactivity(tenantId, deviceId, inactivityTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastInactivityTime(inactivityTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceInactivityMsg(deviceInactivityMsg) - .build(); - } - }, callback); - } - - private void routeEvent(ConnectivityEventInfo eventInfo, TbCallback callback) { - var tenantId = eventInfo.getTenantId(); - var deviceId = eventInfo.getDeviceId(); - long eventTime = eventInfo.getEventTime(); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); - if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) { - log.debug("[{}][{}] Forwarding device connectivity event to local service. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime); - try { - eventInfo.forwardToLocalService(); - } catch (Exception e) { - log.error("[{}][{}] Failed to process device connectivity event. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime, e); - callback.onFailure(e); - return; - } - callback.onSuccess(); - } else { - TransportProtos.ToCoreMsg msg = eventInfo.toQueueMsg(); - log.debug("[{}][{}] Sending device connectivity message to core. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime); - clusterService.pushMsgToCore(tpi, UUID.randomUUID(), msg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure)); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index 23adacfc2e..2cdee7b7d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -161,9 +161,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene @Override public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { onTimeSeriesUpdate(entityId, ts); - if (entityId.getEntityType() == EntityType.DEVICE) { - updateDeviceInactivityTimeout(tenantId, entityId, ts); - } callback.onSuccess(); } @@ -171,9 +168,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene public void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, TbCallback callback) { onTimeSeriesUpdate(entityId, keys.stream().map(key -> new BasicTsKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList())); - if (entityId.getEntityType() == EntityType.DEVICE) { - deleteDeviceInactivityTimeout(tenantId, entityId, keys); - } callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 78d7f01404..9e9ca42e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -436,7 +436,9 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc ctx.sendWsMsg(update); } else { ctx.doFetchAlarmCount(); - ctx.createAlarmSubscriptions(); + if (entitiesIds != null) { + ctx.createAlarmSubscriptions(); + } TbAlarmCountSubCtx finalCtx = ctx; ScheduledFuture task = scheduler.scheduleWithFixedDelay( () -> refreshDynamicQuery(finalCtx), diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java index f6ce2a6a6b..f6b4067543 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java @@ -39,6 +39,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; @@ -359,7 +360,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx { EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityDataSortOrder entitiesSortOrder; if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { - entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); + entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME)); } else { entitiesSortOrder = sortOrder; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 178dde33e2..06fe7f4036 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.util.ThrowingRunnable; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -61,6 +62,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> importServices = new HashMap<>(); private final RelationService relationService; + private final CalculatedFieldService calculatedFieldService; private final RateLimitService rateLimitService; private final TbLogEntityActionService logEntityActionService; @@ -72,7 +74,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE ); - @Override public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { if (!rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, ctx.getTenantId())) { @@ -129,13 +130,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } } - @Override public Comparator getEntityTypeComparatorForImport() { return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); } - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { EntityExportService exportService = exportServices.get(entityType); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 2b7e7d3593..b77447df8d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -32,7 +32,6 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; - void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException; Comparator getEntityTypeComparatorForImport(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index b0bea68290..5c2fcc7dc6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; @@ -59,6 +61,8 @@ public class DefaultEntityExportService ctx, I entityId) throws ThingsboardException { @@ -98,6 +102,10 @@ public class DefaultEntityExportService> attributes = exportAttributes(ctx, entity); exportData.setAttributes(attributes); } + if (ctx.getSettings().isExportCalculatedFields()) { + List calculatedFields = exportCalculatedFields(ctx, entity.getId()); + exportData.setCalculatedFields(calculatedFields); + } } private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { @@ -141,6 +149,19 @@ public class DefaultEntityExportService exportCalculatedFields(EntitiesExportCtx ctx, EntityId entityId) { + List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), entityId); + calculatedFields.forEach(calculatedField -> { + calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, entityId)); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + argument.setRefEntityId(getExternalIdOrElseInternal(ctx, argument.getRefEntityId())); + } + }); + }); + return calculatedFields; + } + protected ID getExternalIdOrElseInternal(EntitiesExportCtx ctx, ID internalId) { if (internalId == null || internalId.isNullUid()) return internalId; var result = ctx.getExternalId(internalId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 00ce23b2e8..7cd4c3aca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -47,7 +47,11 @@ public class AssetImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return assetService.saveAsset(asset); + Asset savedAsset = assetService.saveAsset(asset); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, savedAsset, exportData, idProvider); + } + return savedAsset; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java index 97c54957bf..32a0090a4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java @@ -50,7 +50,11 @@ public class AssetProfileImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return assetProfileService.saveAssetProfile(assetProfile); + AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, saved, exportData, idProvider); + } + return saved; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7044b235a0..bfa95af83c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.HasDefaultOption; import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; @@ -78,6 +80,8 @@ public abstract class BaseEntityImportService existing = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), savedEntity.getId()); + List fieldsToSave = exportData.getCalculatedFields().stream() + .peek(calculatedField -> { + calculatedField.setTenantId(ctx.getTenantId()); + calculatedField.setEntityId(savedEntity.getId()); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), ctx.isFinalImportAttempt())); + } + }); + }).toList(); + + for (CalculatedField existingField : existing) { + boolean found = fieldsToSave.stream().anyMatch(importedField -> compareCalculatedFields(existingField, importedField)); + if (!found) { + calculatedFieldService.deleteCalculatedField(ctx.getTenantId(), existingField.getId()); + updated = true; + } + } + + for (CalculatedField calculatedField : fieldsToSave) { + boolean found = existing.stream().anyMatch(existingField -> compareCalculatedFields(existingField, calculatedField)); + if (!found) { + calculatedFieldService.save(calculatedField); + updated = true; + } + } + return updated; + } + + private boolean compareCalculatedFields(CalculatedField existingField, CalculatedField newField) { + CalculatedField oldCopy = new CalculatedField(existingField); + CalculatedField newCopy = new CalculatedField(newField); + oldCopy.setId(null); + newCopy.setId(null); + oldCopy.setVersion(null); + newCopy.setVersion(null); + oldCopy.setCreatedTime(0); + newCopy.setCreatedTime(0); + return oldCopy.equals(newCopy); + } + protected void onEntitySaved(User user, E savedEntity, E oldEntity) throws ThingsboardException { logEntityActionService.logEntityAction(user.getTenantId(), savedEntity.getId(), savedEntity, null, oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, user); } - @SuppressWarnings("unchecked") protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) { return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) @@ -313,10 +364,10 @@ public abstract class BaseEntityImportService new MissingEntityException(externalId)); } - @SuppressWarnings("unchecked") @RequiredArgsConstructor protected class IdProvider { + private final EntitiesImportCtx ctx; private final EntityImportResult importResult; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 61b9839cbd..84e264efdd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -64,13 +64,18 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return deviceProfileService.saveDeviceProfile(deviceProfile); + DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, saved, exportData, idProvider); + } + return saved; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index eed18abe52..89b51ce84a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -94,7 +94,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.function.Function; -import java.util.stream.Collectors; import static com.google.common.util.concurrent.Futures.transform; import static org.thingsboard.server.common.data.sync.vc.VcUtils.checkBranchName; @@ -304,6 +303,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) + .saveCalculatedFields(config.isLoadCalculatedFields()) .findExistingByName(false) .build()); ctx.setFinalImportAttempt(true); @@ -327,7 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont var sw = TbStopWatch.create("before"); List entityTypes = request.getEntityTypes().keySet().stream() - .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + .sorted(exportImportService.getEntityTypeComparatorForImport()).toList(); for (EntityType entityType : entityTypes) { log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); sw.startNew("Entities " + entityType.name()); @@ -362,6 +362,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) + .saveCalculatedFields(config.isLoadCalculatedFields()) .findExistingByName(config.isFindExistingEntityByName()) .build(); } @@ -471,7 +472,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public ListenableFuture compareEntityDataToVersion(User user, EntityId entityId, String versionId) throws Exception { + public ListenableFuture compareEntityDataToVersion(User user, EntityId entityId, String versionId) { HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); @@ -484,6 +485,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) + .exportCalculatedFields(otherVersion.hasCalculatedFields()) .build()); EntityExportData currentVersion; try { @@ -503,7 +505,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + public ListenableFuture> listBranches(TenantId tenantId) { return gitServiceQueue.listBranches(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java index 669dad347a..b3007c6738 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -69,6 +69,7 @@ public abstract class EntitiesExportCtx { .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) .exportCredentials(config.isSaveCredentials()) + .exportCalculatedFields(config.isSaveCalculatedFields()) .build(); } @@ -85,4 +86,5 @@ public abstract class EntitiesExportCtx { log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); externalIdMap.put(internalId, externalId != null ? externalId : internalId); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index ab836fa13d..e24683cd5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -91,6 +91,10 @@ public class EntitiesImportCtx { return getSettings().isSaveCredentials(); } + public boolean isSaveCalculatedFields() { + return getSettings().isSaveCalculatedFields(); + } + public EntityId getInternalId(EntityId externalId) { var result = externalToInternalIdMap.get(externalId); log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); @@ -140,5 +144,4 @@ public class EntitiesImportCtx { return notFoundIds.contains(externalId); } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java index e0a5b40281..55c8a763b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java @@ -39,6 +39,7 @@ public class EntityTypeExportCtx extends EntitiesExportCtx .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) .exportCredentials(config.isSaveCredentials()) + .exportCalculatedFields(config.isSaveCalculatedFields()) .build(); this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 2bdbd917ab..36cd5516a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -259,8 +259,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) { - } + public void onSuccess(@Nullable Void tmp) {} @Override public void onFailure(Throwable t) { @@ -324,6 +323,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, result -> callback.onSuccess(null), callback::onFailure); } + private void addMainCallback(ListenableFuture saveFuture, Consumer onSuccess) { + addMainCallback(saveFuture, onSuccess, null); + } + private void addMainCallback(ListenableFuture saveFuture, Consumer onSuccess, Consumer onFailure) { DonAsynchron.withCallback(saveFuture, onSuccess, onFailure, tsCallBackExecutor); } @@ -345,13 +348,12 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public void onFailure(Throwable t) { - } + public void onFailure(Throwable t) {} }; } private FutureCallback getCalculatedFieldCallback(FutureCallback> originalCallback, List keys) { - return new FutureCallback() { + return new FutureCallback<>() { @Override public void onSuccess(Void unused) { originalCallback.onSuccess(keys); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 62913a187b..4b60ec923a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -93,7 +93,8 @@ public class TbCoreTransportApiService { @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(transportApiService); + transportApiTemplate.subscribe(); + transportApiTemplate.launch(transportApiService); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java index 90f6b36e0f..7288a8bac9 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java @@ -229,6 +229,7 @@ public class DefaultWebSocketService implements WebSocketService { } catch (TbRateLimitsException e) { log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } catch (Exception e) { + sendError(sessionRef, cmd.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR, e.getMessage()); log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ec9925bf06..2ee3c439c1 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -512,6 +512,8 @@ actors: enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" + # Time in seconds to receive calculation result. + calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" debug: settings: @@ -815,7 +817,7 @@ spring: events: # Enable dedicated datasource (a separate database) for events and audit logs. # Before enabling this, make sure you have set up the following tables in the new DB: - # error_event, lc_event, rule_chain_debug_event, rule_node_debug_event, stats_event, audit_log + # error_event, lc_event, rule_chain_debug_event, rule_node_debug_event, stats_event, audit_log, cf_debug_event enabled: "${SPRING_DEDICATED_EVENTS_DATASOURCE_ENABLED:false}" # Database driver for Spring JPA for events datasource driverClassName: "${SPRING_EVENTS_DATASOURCE_DRIVER_CLASS_NAME:org.postgresql.Driver}" @@ -856,6 +858,7 @@ audit-log: "edge": "${AUDIT_LOG_MASK_EDGE:W}" # Edge logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation "tb_resource": "${AUDIT_LOG_MASK_RESOURCE:W}" # TB resource logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation "ota_package": "${AUDIT_LOG_MASK_OTA_PACKAGE:W}" # Ota package logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "calculated_field": "${AUDIT_LOG_MASK_CALCULATED_FIELD:W}" # Calculated field logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation sink: # Type of external sink. possible options: none, elasticsearch type: "${AUDIT_LOG_SINK_TYPE:none}" @@ -1597,6 +1600,16 @@ queue: - key: max.poll.records # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" + edqs.events: + # Key-value properties for Kafka consumer for edqs.events topic + - key: max.poll.records + # Max poll records for edqs.events topic + value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}" + edqs.state: + # Key-value properties for Kafka consumer for edqs.state topic + - key: max.poll.records + # Max poll records for edqs.state topic + value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1630,6 +1643,12 @@ queue: calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Calculated Field State topics calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1705,6 +1724,44 @@ queue: enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" + edqs: + sync: + # Enable/disable EDQS synchronization + enabled: "${TB_EDQS_SYNC_ENABLED:false}" + # Batch size of entities being synced with EDQS + entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" + # Batch size of timeseries data being synced with EDQS + ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" + api: + # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) + supported: "${TB_EDQS_API_SUPPORTED:false}" + # Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished + auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}" + # Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices) + mode: "${TB_EDQS_MODE:local}" + local: + # Path to RocksDB for EDQS backup when running in local mode + rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:${user.home}/.rocksdb/edqs}" + # Number of partitions for EDQS topics + partitions: "${TB_EDQS_PARTITIONS:12}" + # EDQS partitioning strategy: tenant (partition is resolved by tenant id) or none (no specific strategy, resolving by message key) + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS requests topic + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + # EDQS responses topic + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + # Poll interval for EDQS topics + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + # Maximum amount of pending requests to EDQS + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout for requests to EDQS + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + stats: + # Enable/disable statistics for EDQS + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" + vc: # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 975c3b426b..e90aecb41e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -402,7 +402,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } catch (Exception e) { throw new RuntimeException(e); } - Awaitility.await("all tasks processed").atMost(60, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) + Awaitility.await("all tasks processed").atMost(90, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) .until(() -> storage.getLag("tb_housekeeper") == 0); } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java new file mode 100644 index 0000000000..ce5221ab89 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.state.EdqsStateService; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { +// "queue.type=kafka", // uncomment to use Kafka +// "queue.kafka.bootstrap.servers=10.7.1.254:9092", + "queue.edqs.sync.enabled=true", + "queue.edqs.api.supported=true", + "queue.edqs.api.auto_enable=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { + + @Autowired + private EdqsApiService edqsApiService; + + @Autowired + private EdqsStateService edqsStateService; + + @MockBean // so that we don't do backup for tests + private EdqsRocksDb edqsRocksDb; + + @Before + public void before() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled() && edqsStateService.isReady()); + } + + @Override + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == expectedResult); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index ac618d7f96..011399e883 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.awaitility.Awaitility; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -49,6 +51,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; @@ -70,9 +73,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.BiPredicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @@ -130,36 +135,25 @@ public class EntityQueryControllerTest extends AbstractControllerTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); - countQuery = new EntityCountQuery(entityListFilter); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); EntityTypeFilter filter2 = new EntityTypeFilter(); filter2.setEntityType(EntityType.DEVICE); - - EntityCountQuery countQuery2 = new EntityCountQuery(filter2); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class); - Assert.assertEquals(97, count2.longValue()); + countQuery = new EntityCountQuery(filter2); + countByQueryAndCheck(countQuery, 97); } @Test @@ -169,51 +163,44 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); allDeviceFilter.setEntityType(EntityType.DEVICE); EntityCountQuery query = new EntityCountQuery(allDeviceFilter); - Long initialCount = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); + countByQueryAndCheck(query, 0); loginTenantAdmin(); List devices = new ArrayList<>(); + String devicePrefix = "Device" + RandomStringUtils.randomAlphabetic(5); for (int i = 0; i < 97; i++) { Device device = new Device(); - device.setName("Device" + i); + device.setName(devicePrefix + i); device.setType("default"); device.setLabel("testLabel" + (int) (Math.random() * 1000)); devices.add(doPost("/api/device", device, Device.class)); Thread.sleep(1); } DeviceTypeFilter filter = new DeviceTypeFilter(); - filter.setDeviceType("default"); + filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter(""); loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); + countByQueryAndCheck(countQuery, 97); - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); - - filter.setDeviceType("unknown"); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); - - filter.setDeviceType("default"); - filter.setDeviceNameFilter("Device1"); + filter.setDeviceTypes(List.of("unknown")); + countByQueryAndCheck(countQuery, 0); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(devicePrefix + "1"); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); + countByQueryAndCheck(countQuery, 97); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); - Assert.assertEquals(initialCount + 97, count2.longValue()); + countByQueryAndCheck(countQuery, 97); } @Test @@ -371,11 +358,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -383,8 +366,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -406,8 +388,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -423,9 +404,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null); - PageData data2 = - doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference>() { - }); + PageData data2 = findByQuery(query2); Assert.assertEquals(97, data2.getTotalElements()); Assert.assertEquals(10, data2.getTotalPages()); @@ -473,20 +452,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(10, data.getTotalElements()); + findByQueryAndCheck(query, 10); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); } private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) { @@ -531,14 +505,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -551,6 +523,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(45)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); @@ -559,13 +532,11 @@ public class EntityQueryControllerTest extends AbstractControllerTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -604,6 +575,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); DynamicValue dynamicValue = @@ -627,16 +599,16 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - Awaitility.await() + await() .alias("data by query") .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == numOfDevices; }); - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(numOfDevices, loadedEntities.size()); @@ -694,11 +666,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -712,9 +680,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { }); EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); } @Test @@ -742,28 +708,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); // all devices with ownerName = TEST TENANT - EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); - checkEntitiesCount(query, numOfDevices); + EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == numOfDevices); // all devices with ownerName = TEST TENANT - EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); - checkEntitiesCount(activeAlarmTimeToLongQuery, 0); + EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); + countByQueryAndCheck(activeAlarmTimeToLongQuery, 0); // all devices with wrong ownerName EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter)); - checkEntitiesCount(wrongTenantNameQuery, 0); + countByQueryAndCheck(wrongTenantNameQuery, 0); // all devices with owner type = TENANT EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter)); - checkEntitiesCount(tenantEntitiesQuery, numOfDevices); + countByQueryAndCheck(tenantEntitiesQuery, numOfDevices); // all devices with owner type = CUSTOMER EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter)); - checkEntitiesCount(customerEntitiesQuery, 0); + countByQueryAndCheck(customerEntitiesQuery, 0); } @Test @@ -790,7 +757,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); EntityDataSortOrder sortOrder = new EntityDataSortOrder( @@ -851,41 +818,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(1, data.getTotalElements()); - Assert.assertEquals(1, data.getTotalPages()); - Assert.assertEquals(1, data.getData().size()); + findByQueryAndCheck(query, 1); // unnassign dashboard login(TENANT_EMAIL, TENANT_PASSWORD); doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD); - PageData dataAfterUnassign = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(0, dataAfterUnassign.getTotalElements()); - Assert.assertEquals(0, dataAfterUnassign.getTotalPages()); - Assert.assertEquals(0, dataAfterUnassign.getData().size()); + findByQueryAndCheck(query, 0); } private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { - Awaitility.await() + await() .alias("data by query") .atMost(30, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == expectedNumOfDevices; }); - if (expectedNumOfDevices == 0) { - return; - } - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + if (expectedNumOfDevices == 0) { + return; + } + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(expectedNumOfDevices, loadedEntities.size()); @@ -898,25 +853,37 @@ public class EntityQueryControllerTest extends AbstractControllerTest { String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue(); Assert.assertEquals("Device" + i, name); - Assert.assertEquals( expectedOwnerName, ownerName); - Assert.assertEquals( expectedOwnerType, ownerType); + Assert.assertEquals(expectedOwnerName, ownerName); + Assert.assertEquals(expectedOwnerType, ownerType); Assert.assertEquals("1" + i, alarmActiveTime); } } - private void checkEntitiesCount(EntityCountQuery query, int expectedNumOfDevices) { - Awaitility.await() - .alias("count by query") - .atMost(30, TimeUnit.SECONDS) - .until(() -> { - var count = doPost("/api/entitiesQuery/count", query, Integer.class); - return count == expectedNumOfDevices; - }); - } + protected PageData findByQuery(EntityDataQuery query) throws Exception { + return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() { + }); + } + + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception { + PageData result = findByQuery(query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected Long countByQuery(EntityCountQuery countQuery) throws Exception { + return doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + } + + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) throws Exception { + Long result = countByQuery(query); + assertThat(result).isEqualTo(expectedResult); + return result; + } private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) { KeyFilter tenantOwnerNameFilter = new KeyFilter(); tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName)); + tenantOwnerNameFilter.setValueType(EntityKeyValueType.STRING); StringFilterPredicate ownerNamePredicate = new StringFilterPredicate(); ownerNamePredicate.setValue(FilterPredicateValue.fromString(value)); ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL); @@ -927,6 +894,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) { KeyFilter numericFilter = new KeyFilter(); numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute)); + numericFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(value)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index 69d50c61d1..9801907d3b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "server.ws.alarms_per_alarm_status_subscription_cache_size=5" + "server.ws.alarms_per_alarm_status_subscription_cache_size=5", + "server.ws.dynamic_page_link.refresh_interval=15" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 17d0f3fe84..4a303e5253 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -153,7 +153,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()); + ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()).get(0); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -308,7 +308,7 @@ public class HashPartitionServiceTest { partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID); - return event.getPartitionsMap().get(queueKey).size() == systemQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == systemQueue.getPartitions(); }); Mockito.reset(applicationEventPublisher); @@ -336,14 +336,14 @@ public class HashPartitionServiceTest { // expecting event about no partitions for isolated queue key verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); partitionService_dedicated.updateQueues(List.of(queueUpdateMsg)); partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).size() == isolatedQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == isolatedQueue.getPartitions(); }); @@ -361,7 +361,7 @@ public class HashPartitionServiceTest { partitionService_dedicated.removeQueues(List.of(queueDeleteMsg)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); } @@ -381,12 +381,12 @@ public class HashPartitionServiceTest { Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> { List queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10)) .map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId)) - .limit(100).collect(Collectors.toList()); + .limit(100).toList(); for (int partition = 0; partition < 10; partition++) { - ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()); + ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()).get(0); for (QueueKey queueKey : queues) { - ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()); + ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()).get(0); assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId()) .isEqualTo(expectedAssignedRuleEngine); } @@ -434,6 +434,7 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge"); ReflectionTestUtils.setField(partitionService, "edgePartitions", 10); + ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12); partitionService.init(); partitionService.partitionsInit(); return partitionService; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 2cced15f59..320093e8f4 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -90,7 +90,7 @@ public class ScriptCalculatedFieldStateTest { state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); - boolean stateUpdated = state.updateState(newArgs); + boolean stateUpdated = state.updateState(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -107,7 +107,7 @@ public class ScriptCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); Map newArgs = Map.of("assetHumidity", newArgEntry); - boolean stateUpdated = state.updateState(newArgs); + boolean stateUpdated = state.updateState(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -191,7 +191,7 @@ public class ScriptCalculatedFieldStateTest { config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); - config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}"); + config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity}"); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index c06e835937..f0059e4f6f 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; @@ -87,7 +88,7 @@ public class SimpleCalculatedFieldStateTest { )); Map newArgs = Map.of("key3", key3ArgEntry); - boolean stateUpdated = state.updateState(newArgs); + boolean stateUpdated = state.updateState(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( @@ -105,7 +106,7 @@ public class SimpleCalculatedFieldStateTest { SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); Map newArgs = Map.of("key1", newArgEntry); - boolean stateUpdated = state.updateState(newArgs); + boolean stateUpdated = state.updateState(ctx, newArgs); assertThat(stateUpdated).isTrue(); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); @@ -119,7 +120,7 @@ public class SimpleCalculatedFieldStateTest { )); Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); - assertThatThrownBy(() -> state.updateState(newArgs)) + assertThatThrownBy(() -> state.updateState(ctx, newArgs)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @@ -138,7 +139,7 @@ public class SimpleCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0))); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49))); } @Test @@ -154,6 +155,26 @@ public class SimpleCalculatedFieldStateTest { .hasMessage("Argument 'key2' is not a number."); } + @Test + void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("key1", 11.3456), 145L), + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new DoubleDataEntry("key2", 15.1), 165L), + "key3", new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new DoubleDataEntry("key3", 23.1), 184L) + )); + + Output output = getCalculatedFieldConfig().getOutput(); + output.setDecimalsByDefault(3); + ctx.setOutput(output); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.546))); + } + @Test void testIsReadyWhenNotAllArgPresent() { assertThat(state.isReady()).isFalse(); @@ -219,6 +240,7 @@ public class SimpleCalculatedFieldStateTest { output.setName("output"); output.setType(OutputType.ATTRIBUTES); output.setScope(AttributeScope.SERVER_SCOPE); + output.setDecimalsByDefault(0); config.setOutput(output); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index e83e30663d..2c48ed9167 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -71,6 +71,6 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenValueWasNotChanged() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue(); } } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java new file mode 100644 index 0000000000..50c80d08c7 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -0,0 +1,128 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy; + +import com.google.common.collect.Lists; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.IdBased; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.edqs.sync.enabled=true", + "queue.edqs.api.supported=true", + "queue.edqs.api.auto_enable=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityServiceTest extends EntityServiceTest { + + @Autowired + private EdqsApiService edqsApiService; + + @MockBean + private EdqsRocksDb edqsRocksDb; + + @Before + public void beforeEach() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled()); + } + + // sql implementation has a bug with data duplication, edqs implementation returns correct value + @Override + @Test + public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { + List buildings = new ArrayList<>(); + List apartments = new ArrayList<>(); + Map> entityNameByTypeMap = new HashMap<>(); + Map childParentRelationMap = new HashMap<>(); + createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap); + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setMultiRoot(true); + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(buildings.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.FROM); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + countByQueryAndCheck(countQuery, 63); + + filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 27); + + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.TO); + filter.setFilters(Lists.newArrayList( + new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), + new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 3); + + deviceService.deleteDevicesByTenantId(tenantId); + assetService.deleteAssetsByTenantId(tenantId); + } + + @Override + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), + loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries)); + } + + @Override + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); + } + + @Override + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query), + result -> result == expectedResult); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java similarity index 80% rename from dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java rename to application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 77f9a6bc23..18687cedba 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -13,22 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.service; +package org.thingsboard.server.service.entitiy; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; @@ -37,6 +40,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -47,6 +51,7 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; @@ -64,6 +69,7 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; @@ -76,17 +82,22 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardDao; +import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewDao; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; @@ -106,13 +117,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @Slf4j @DaoSqlTest -public class EntityServiceTest extends AbstractServiceTest { +public class EntityServiceTest extends AbstractControllerTest { static final int ENTITY_COUNT = 5; public static final String TEST_CUSTOMER_NAME = "Test"; @@ -120,6 +130,12 @@ public class EntityServiceTest extends AbstractServiceTest { @Autowired AssetService assetService; @Autowired + AssetProfileService assetProfileService; + @Autowired + DashboardService dashboardService; + @Autowired + EntityViewService entityViewService; + @Autowired UserService userService; @Autowired AttributesService attributesService; @@ -158,7 +174,7 @@ public class EntityServiceTest extends AbstractServiceTest { } @Test - public void testCountEntitiesByQuery() throws InterruptedException { + public void testCountEntitiesByQuery() { List devices = new ArrayList<>(); for (int i = 0; i < 97; i++) { Device device = new Device(); @@ -174,33 +190,26 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); deviceService.deleteDevicesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } - @Test public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { List assets = new ArrayList<>(); @@ -212,19 +221,15 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(31, count); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) + countByQueryAndCheck(countQuery, 31); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter.setRootEntity(devices.get(0).getId()); filter.setDirection(EntitySearchDirection.TO); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Manages", Collections.singletonList(EntityType.TENANT)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(1, count); + countByQueryAndCheck(countQuery, 1); DeviceSearchQueryFilter filter2 = new DeviceSearchQueryFilter(); filter2.setRootEntity(tenantId); @@ -232,18 +237,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter2.setRelationType("Contains"); countQuery = new EntityCountQuery(filter2); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter2.setDeviceTypes(Arrays.asList("default0", "default1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(10, count); + countByQueryAndCheck(countQuery, 10); filter2.setRootEntity(devices.get(0).getId()); filter2.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); AssetSearchQueryFilter filter3 = new AssetSearchQueryFilter(); filter3.setRootEntity(tenantId); @@ -251,18 +252,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter3.setRelationType("Manages"); countQuery = new EntityCountQuery(filter3); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter3.setAssetTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); filter3.setRootEntity(devices.get(0).getId()); filter3.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -279,11 +276,12 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData entityDataByQuery = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData entityDataByQuery = findByQueryAndCheck(query, 5); List data = entityDataByQuery.getData(); Assert.assertEquals(data.size(), 5); data.forEach(entityData -> Assert.assertNotNull(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("phone"))); + countByQueryAndCheck(query, 5); } private void createTestUserRelations(TenantId tenantId, List users) { @@ -313,30 +311,24 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setEdgeNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setEdgeTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setEdgeTypes(List.of("default")); filter.setEdgeNameFilter("Edge1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.EDGE); entityListFilter.setEntityList(edges.stream().map(Edge::getId).map(EdgeId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); edgeService.deleteEdgesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -361,13 +353,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setRelationType("Manages"); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter.setEdgeTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); } private Edge createEdge(int i, String type) { @@ -424,19 +413,10 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(25, loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -447,23 +427,10 @@ public class EntityServiceTest extends AbstractServiceTest { highTemperatureFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(highTemperatureFilter); - query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -483,13 +450,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(63, count); + countByQueryAndCheck(countQuery, 63); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(27, count); + countByQueryAndCheck(countQuery, 27); filter.setMultiRootEntitiesType(EntityType.ASSET); filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); @@ -497,13 +461,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(9, count); + countByQueryAndCheck(countQuery, 9); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); - } @Test @@ -539,15 +500,6 @@ public class EntityServiceTest extends AbstractServiceTest { onlineStatusFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(onlineStatusFilter); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - long expectedEntitiesCnt = entityNameByTypeMap.entrySet() .stream() .filter(e -> !e.getKey().equals("building")) @@ -555,6 +507,14 @@ public class EntityServiceTest extends AbstractServiceTest { .map(Map.Entry::getValue) .filter(e -> StringUtils.endsWith(e, "_1")) .count(); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + PageData data = findByQueryAndCheck(query, expectedEntitiesCnt); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } Assert.assertEquals(expectedEntitiesCnt, loadedEntities.size()); Map actualRelations = new HashMap<>(); @@ -603,20 +563,12 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(25, loadedEntities.size()); + List loadedEntities = findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceTemperatures); + loadedEntities.forEach(entity -> Assert.assertTrue(devices.stream().map(Device::getId).collect(Collectors.toSet()).contains(entity.getEntityId()))); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -629,21 +581,8 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -677,18 +616,9 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "consumption")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(5, loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); + List deviceTemperatures = consumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "consumption", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -701,21 +631,8 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highConsumptions.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highConsumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "consumption", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -897,9 +814,7 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -907,7 +822,7 @@ public class EntityServiceTest extends AbstractServiceTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -932,7 +847,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -946,11 +861,12 @@ public class EntityServiceTest extends AbstractServiceTest { devices.get(1).setLabel(null); devices.forEach(deviceService::saveDevice); + // FIXME (for Dasha, plz investigate): + // this and other tests below submit an empty value to a KEY FILTER, this is not "search text". + // why are we supposed to ignore it and return all devices? maybe it's a bug? String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -962,9 +878,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = devices.get(2).getLabel(); EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size() - 1, result.getTotalElements()); + findByQueryAndCheck(query, devices.size() - 1); } @Test @@ -977,8 +891,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -990,9 +903,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.STARTS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1004,9 +915,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.ENDS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1018,9 +927,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1032,9 +939,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = "label-"; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(2, result.getTotalElements()); + findByQueryAndCheck(query, 2); } @Test @@ -1046,9 +951,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1072,34 +975,27 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test public void testFindEntityDataByQuery_filter_entity_name_ends_with() { List devices = new ArrayList<>(); + String suffixes = RandomStringUtils.randomAlphanumeric(5); for (int i = 0; i < 10; i++) { Device device = new Device(); device.setTenantId(tenantId); - device.setName("Device " + i + " test"); + device.setName("Device " + i + suffixes); device.setType("default"); devices.add(device); } @@ -1108,29 +1004,21 @@ public class EntityServiceTest extends AbstractServiceTest { EntityNameFilter deviceTypeFilter = new EntityNameFilter(); deviceTypeFilter.setEntityType(EntityType.DEVICE); - deviceTypeFilter.setEntityNameFilter("%test"); + deviceTypeFilter.setEntityNameFilter("%" + suffixes); EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + findByQueryAndCheck(query, devices.size()); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter("%" + suffixes + "%"); + findByQueryAndCheck(query, devices.size()); - deviceTypeFilter.setEntityNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); - - deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter(suffixes + "%"); + findByQueryAndCheck(query, 0); - deviceTypeFilter.setEntityNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter(suffixes); + findByQueryAndCheck(query, 0); } @Test @@ -1154,19 +1042,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setEntityNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1190,24 +1072,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1233,31 +1107,12 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); + PageData result = findByQueryAndCheck(query, 1); String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName).isEqualTo(devices.get(0).getName()); } - @Test - public void testFindEntitiesByApiUsageStateFilter() { - apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); - ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); - apiUsageStateFilter.setCustomerId(customerId); - - List entityFields = List.of( - new EntityKey(EntityKeyType.ENTITY_FIELD, "name") - ); - - EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); - EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); - String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(name).isEqualTo(TEST_CUSTOMER_NAME); - } - @Test public void testFindEntitiesByRelationEntityTypeFilter() { Customer customer = new Customer(); @@ -1313,11 +1168,8 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setRootEntity(asset.getId()); EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - PageData relationsResult = entityService.findEntityDataByQuery(tenantId, customer.getId(), query); - long relationsResultCnt = entityService.countEntitiesByQuery(tenantId, customer.getId(), query); - - Assert.assertEquals(relationsCnt, relationsResult.getData().size()); - Assert.assertEquals(relationsCnt, relationsResultCnt); + findByQueryAndCheck(customer.getId(), query, relationsCnt); + countByQueryAndCheck(customer.getId(), query, relationsCnt); } } @@ -1342,24 +1194,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1383,19 +1227,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1419,24 +1257,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1460,24 +1290,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1489,6 +1311,7 @@ public class EntityServiceTest extends AbstractServiceTest { asset.setTenantId(tenantId); asset.setName("Asset test" + i); asset.setType("default"); + asset.setAssetProfileId(assetProfileService.findDefaultAssetProfile(tenantId).getId()); assets.add(asset); } @@ -1501,25 +1324,105 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("%test"); + findByQueryAndCheck(query, 0); + } + + @Test + public void testFindEntitiesBySingleEntityFilter_customer() { + List customerDevices = new ArrayList<>(); + List tenantDevices = new ArrayList<>(); + + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test" + i); + device.setType("default"); + Device saved = deviceService.saveDevice(device); + customerDevices.add(saved); + } + + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Tenant test device" + i); + device.setType("default"); + tenantDevices.add(deviceService.saveDevice(device)); + } + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(customerDevices.get(0).getId()); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + PageData result = findByQueryAndCheck(query, 1); + String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); + + // find by customer user with generic permission + PageData customerResults = findByQueryAndCheck(customerId, query, 1); + + String cutomerDeviceName = customerResults.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(cutomerDeviceName).isEqualTo(customerDevices.get(0).getName()); + + // try to find tenant device by customer user + SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); + tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); + EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); + findByQueryAndCheck(customerId, customerQuery2, 0); } - private PageData searchEntities(EntityDataQuery query) { - return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + private List getResultDeviceIds(PageData result) { + return result.getData().stream().map(entityData -> (DeviceId) entityData.getEntityId()).collect(Collectors.toList()); } + private Device createDevice(CustomerId customerId) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test " + RandomStringUtils.randomAlphabetic(5)); + device.setType("default"); + return device; + } + + @Test + public void testFindEntitiesByApiUsageStateFilter() { + ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); + + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); + PageData result = findByQueryAndCheck(query, 1); + String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(name).isEqualTo(TEST_TENANT_NAME); + + // find by customer user with generic permissions + apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); + PageData customerResult = findByQueryAndCheck(customerId, query, 1); + + String customerResultName = customerResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(customerResultName).isEqualTo(TEST_CUSTOMER_NAME); + + // find by tenant user with customerId filter + apiUsageStateFilter.setCustomerId(customerId); + PageData tenantResult = findByQueryAndCheck(query, 1); + String tenantResultName = tenantResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); + } + + private EntityDataQuery createDeviceSearchQuery(String deviceField, StringOperation operation, String searchQuery) { DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); deviceTypeFilter.setDeviceTypes(List.of("default")); @@ -1598,45 +1501,15 @@ public class EntityServiceTest extends AbstractServiceTest { for (EntityKeyType currentAttributeKeyType : attributesEntityTypes) { List latestValues = Collections.singletonList(new EntityKey(currentAttributeKeyType, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(67, loadedEntities.size()); - List loadedTemperatures = new ArrayList<>(); - for (Device device : devices) { - loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) - .getLatest().get(currentAttributeKeyType).get("temperature").getValue()); - } - List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).toList(); + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45); List keyFiltersHighTemperature = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersHighTemperature); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList()); - List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); - + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", highTemperatures.stream().map(Object::toString).toList()); } deviceService.deleteDevicesByTenantId(tenantId); } @@ -1716,89 +1589,48 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFiltersNotEqualTemperature = Collections.singletonList(notEqualTemperatureFilter); //Greater Operation + List deviceTemperatures = greaterTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterTemperature); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(greaterTemperatures.size(), loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - List deviceTemperatures = greaterTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Greater or equal Operation + deviceTemperatures = greaterOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(greaterOrEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = greaterOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Less Operation + deviceTemperatures = lessTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(lessTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = lessTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Less or equal Operation + deviceTemperatures = lessOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(lessOrEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = lessOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Equal Operation + deviceTemperatures = equalTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(equalTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = equalTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Not equal Operation + deviceTemperatures = notEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(notEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = notEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); - + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -1843,24 +1675,10 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(67, loadedEntities.size()); - List loadedTemperatures = new ArrayList<>(); - for (Device device : devices) { - loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) - .getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); - } List deviceTemperatures = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -1871,23 +1689,10 @@ public class EntityServiceTest extends AbstractServiceTest { highTemperatureFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(highTemperatureFilter); - query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -1995,7 +1800,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, equalStrings.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(equalStrings.size(), loadedEntities.size()); @@ -2008,7 +1813,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notEqualStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notEqualStrings.size(), loadedEntities.size()); @@ -2021,7 +1826,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, startsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(startsWithStrings.size(), loadedEntities.size()); @@ -2034,7 +1839,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, endsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(endsWithStrings.size(), loadedEntities.size()); @@ -2047,7 +1852,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, containsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(containsStrings.size(), loadedEntities.size()); @@ -2060,7 +1865,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notContainsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notContainsStrings.size(), loadedEntities.size()); @@ -2073,7 +1878,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, deviceTypeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2118,7 +1923,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2133,7 +1938,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2146,7 +1951,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2159,7 +1964,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2172,7 +1977,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2185,7 +1990,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2233,7 +2038,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, deviceTypeFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2241,7 +2046,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, createdTimeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2249,7 +2054,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, null); query = new EntityDataQuery(filter, pageLink, entityFields, null, nameFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2297,12 +2102,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing EntityDataPageLink originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); EntityDataQuery originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, deviceTypeFilters); - PageData originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + PageData originalData = findByQueryAndCheck(originalQuery, expectedDevicesSize); // query without textSearch - optimization is performing EntityDataPageLink optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters); - PageData optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + PageData optimizedData = findByQueryAndCheck(optimizedQuery, expectedDevicesSize); List loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2326,12 +2131,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, attributeFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, attributeFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2355,12 +2160,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, nameFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2388,10 +2193,9 @@ public class EntityServiceTest extends AbstractServiceTest { private List getLoadedEntities(PageData data, EntityDataQuery query) { List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } return loadedEntities; @@ -2422,13 +2226,13 @@ public class EntityServiceTest extends AbstractServiceTest { private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { @@ -2437,10 +2241,10 @@ public class EntityServiceTest extends AbstractServiceTest { tsKv.setDoubleValue(value); KvEntry telemetryValue = new DoubleDataEntry(key, value); BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); - return timeseriesService.save(SYSTEM_TENANT_ID, entityId, timeseries); + return timeseriesService.save(tenantId, entityId, timeseries); } - private void createMultiRootHierarchy(List buildings, List apartments, + protected void createMultiRootHierarchy(List buildings, List apartments, Map> entityNameByTypeMap, Map childParentRelationMap) throws InterruptedException { for (int k = 0; k < 3; k++) { @@ -2511,4 +2315,93 @@ public class EntityServiceTest extends AbstractServiceTest { } } } + + @Test + public void testFindEntitiesWithEntityViewFilter() { + EntityView entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setCustomerId(customerId); + entityView.setName("test"); + entityView.setType("default"); + entityView.setEntityId(new DeviceId(UUID.randomUUID())); + entityView.setKeys(new TelemetryEntityView(List.of("test"), null)); + entityView.setStartTimeMs(124); + entityView.setEndTimeMs(256); + entityView.setExternalId(new EntityViewId(UUID.randomUUID())); + entityView.setAdditionalInfo(JacksonUtil.newObjectNode().put("test", "test")); + entityView = entityViewService.saveEntityView(entityView); + + EntityViewTypeFilter entityViewTypeFilter = new EntityViewTypeFilter(); + entityViewTypeFilter.setEntityViewNameFilter("test"); + entityViewTypeFilter.setEntityViewTypes(List.of("non-existing", "default")); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataQuery query = new EntityDataQuery(entityViewTypeFilter, pageLink, entityFields, Collections.emptyList(), null); + + PageData relationsResult = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 1); + assertThat(relationsResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).isEqualTo(entityView.getName()); + + // find with non existing name + entityViewTypeFilter.setEntityViewNameFilter("non-existing"); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); + + // find with non existing type + entityViewTypeFilter.setEntityViewNameFilter(null); + entityViewTypeFilter.setEntityViewTypes(Collections.singletonList("non-existing")); + + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); + } + + protected PageData findByQuery(EntityDataQuery query) { + return findByQuery(new CustomerId(CustomerId.NULL_UUID), query); + } + + protected PageData findByQuery(CustomerId customerId, EntityDataQuery query) { + return entityService.findEntityDataByQuery(tenantId, customerId, query); + } + + protected PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { + return findByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), query, expectedResultSize); + } + + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + PageData result = entityService.findEntityDataByQuery(tenantId, customerId, query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetry) { + List loadedEntities = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry); + List entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList(); + assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry); + return loadedEntities; + } + + protected List findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + PageData data = findByQueryAndCheck(query, expectedTelemetries.size()); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } + return loadedEntities; + } + + protected long countByQuery(CustomerId customerId, EntityCountQuery query) { + return entityService.countEntitiesByQuery(tenantId, customerId, query); + } + + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); + } + + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + long result = countByQuery(customerId, query); + assertThat(result).isEqualTo(expectedResult); + return result; + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java index fe222237c6..26832f6176 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java @@ -532,6 +532,98 @@ public class DefaultTbCoreConsumerServiceTest { then(statsMock).should(never()).log(inactivityMsg); } + @Test + public void givenProcessingSuccess_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnSuccessCallbackIsCalled() { + // GIVEN + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(stateServiceMock).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time); + then(tbCallbackMock).should().onSuccess(); + then(tbCallbackMock).should(never()).onFailure(any()); + } + + @Test + public void givenProcessingFailure_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnFailureCallbackIsCalled() { + // GIVEN + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + var runtimeException = new RuntimeException("Something bad happened!"); + doThrow(runtimeException).when(stateServiceMock).onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(tbCallbackMock).should(never()).onSuccess(); + then(tbCallbackMock).should().onFailure(runtimeException); + } + + @Test + public void givenStatsEnabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreRecorded() { + // GIVEN + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock); + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", true); + + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(statsMock).should().log(inactivityTimeoutUpdateMsg); + } + + @Test + public void givenStatsDisabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreNotRecorded() { + // GIVEN + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock); + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", false); + + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(statsMock).should(never()).log(inactivityTimeoutUpdateMsg); + } + @Test public void givenRestApiCallResponseMsgProto_whenForwardToRuleEngineCallService_thenCallOnQueueMsg() { // GIVEN @@ -545,4 +637,5 @@ public class DefaultTbCoreConsumerServiceTest { // THEN then(ruleEngineCallServiceMock).should().onQueueMsg(restApiCallResponseMsgProto, tbCallbackMock); } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 8141b924f6..314c76b058 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -626,8 +626,7 @@ public class TbRuleEngineQueueConsumerManagerTest { .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); verify(consumer, times(1)).subscribe(any()); verify(consumer).subscribe(eq(expectedPartitions)); - verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() - .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList())))); + verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions))); verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); verify(consumer, never()).unsubscribe(); @@ -743,9 +742,11 @@ public class TbRuleEngineQueueConsumerManagerTest { } @Override - protected void doSubscribe(List topicNames) { - log.debug("doSubscribe({})", topicNames); - this.topics = topicNames; + protected void doSubscribe(Set partitions) { + this.topics = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toList()); + log.debug("doSubscribe({})", topics); subscribed = true; } diff --git a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java index 8c8b82928a..bf86f98e6e 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java @@ -21,6 +21,7 @@ import org.thingsboard.script.api.tbel.TbDate; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.Comparator; @@ -146,6 +147,31 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { assertEquals(expected, actual); } + @Test + public void mapsImplicitIterationWithoutEntrySet() throws ExecutionException, InterruptedException { + msgStr = msgMapStr; + decoderStr = """ + foreach(element : msg){ + if(element.getKey() == null){ + return raiseError("Bad getKey"); + } + if(element.key == null){ + return raiseError("Bad key"); + } + if(element.getValue() == null){ + return raiseError("Bad getValue"); + } + if(element.value == null){ + return raiseError("Bad value"); + } + } + return msg; + """; + LinkedHashMap expected = expectedMap; + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + } + @Test public void mapsGetInfoSize_Test() throws ExecutionException, InterruptedException { msgStr = msgMapStr; @@ -660,14 +686,13 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { """; decoderStr = """ var list = msg.list; - var listAdd = ["thigsboard", 4, 67]; return { list: list.clone(), length: list.length(), memorySize: list.memorySize(), indOf1: list.indexOf("B", 1), indOf2: list.indexOf(2, 2), - sStr: list.validateClazzInArrayIsOnlyString() + sStr: list.validateClazzInArrayIsOnlyNumber() } """; ArrayList list = new ArrayList<>(List.of(67, 2, 2, 2)); @@ -677,7 +702,7 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { expected.put("memorySize", 32L); expected.put("indOf1", -1); expected.put("indOf2", 2); - expected.put("sStr", false); + expected.put("sStr", true); Object actual = invokeScript(evalScript(decoderStr), msgStr); assertEquals(expected, actual); } @@ -2148,6 +2173,177 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { assertEquals(expected, actual); } + @Test + public void toUnmodifiableExecutionArrayList_Test() throws ExecutionException, InterruptedException { + msgStr = "{}"; + decoderStr = String.format(""" + var original = []; + original.add(0x35); + var unmodifiable = original.toUnmodifiable(); + msg.result = unmodifiable; + return {msg: msg}; + """); + LinkedHashMap expected = new LinkedHashMap<>(); + List expectedList = Arrays.asList(0x35); + LinkedHashMap expectedResult = new LinkedHashMap<>(); + expectedResult.put("result", expectedList); + expected.put("msg", expectedResult); + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + + decoderStr = String.format(""" + var original = []; + original.add(0x67); + var unmodifiable = original.toUnmodifiable(); + unmodifiable.add(0x35); + msg.result = unmodifiable; + return {msg: msg}; + """); + assertThatThrownBy(() -> { + invokeScript(evalScript(decoderStr), msgStr); + }).hasMessageContaining("Error: unmodifiable.add(0x35): List is unmodifiable"); + } + + + @Test + public void toUnmodifiableExecutionHashMap_Test() throws ExecutionException, InterruptedException { + msgStr = "{}"; + decoderStr = String.format(""" + var original = {}; + original.putIfAbsent("entry1", 73); + var unmodifiable = original.toUnmodifiable(); + msg.result = unmodifiable; + return {msg: msg}; + """); + LinkedHashMap expected = new LinkedHashMap<>(); + LinkedHashMap expectedMap = new LinkedHashMap<>(Map.of("entry1", 73)); + LinkedHashMap expectedResult = new LinkedHashMap<>(); + expectedResult.put("result", expectedMap); + expected.put("msg", expectedResult); + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + + decoderStr = String.format(""" + var original = {}; + original.humidity = 73; + var unmodifiable = original.toUnmodifiable(); + unmodifiable.put("temperature1", 96); + msg.result = unmodifiable; + return {msg: msg}; + """); + assertThatThrownBy(() -> { + invokeScript(evalScript(decoderStr), msgStr); + }).hasMessageContaining("Error: unmodifiable.put(\"temperature1\", 96): Map is unmodifiable"); + } + + @Test + public void tbDateFunction_Test() throws ExecutionException, InterruptedException { + String stringDateUTC = "2024-01-01T10:00:00.00Z"; + TbDate d = new TbDate(stringDateUTC); + + msgStr = "{}"; + decoderStr = String.format(""" + var d = new Date("%s"); // TZ => "UTC" + var dIsoY1 = d.toISOString(); // return 2024-01-01T10:00:00Z + d.addYears(1); + var dIsoY2 = d.toISOString(); // return 2025-01-01T10:00:00Z + d.addYears(-2); + var dIsoY3 = d.toISOString(); // return 2023-01-01T10:00:00Z + d.addMonths(2); + var dIsoM1 = d.toISOString(); // return 2023-03-01T10:00:00Z + d.addMonths(10); + var dIsoM2 = d.toISOString(); // return 2024-01-01T10:00:00Z + d.addMonths(-13); + var dIsoM3 = d.toISOString(); // return 2022-12-01T10:00:00Z + d.addWeeks(4); + var dIsoW1 = d.toISOString(); // return 2022-12-29T10:00:00Z + d.addWeeks(-5); + var dIsoW2 = d.toISOString(); // return 2022-11-24T10:00:00Z + d.addDays(6); + var dIsoD1 = d.toISOString(); // return 2022-11-30T10:00:00Z + d.addDays(45); + var dIsoD2 = d.toISOString(); // return 2023-01-14T10:00:00Z + d.addDays(-50); + var dIsoD3 = d.toISOString(); // return 2022-11-25T10:00:00Z + d.addHours(23); + var dIsoH1 = d.toISOString(); // return 2022-11-26T09:00:00Z + d.addHours(-47); + var dIsoH2 = d.toISOString(); // return 2022-11-24T10:00:00Z + d.addMinutes(59); + var dIsoMin1 = d.toISOString(); // return 2022-11-24T10:59:00Z + d.addMinutes(-60); + var dIsoMin2 = d.toISOString(); // return 2022-11-24T09:59:00Z + d.addSeconds(59); + var dIsoS1 = d.toISOString(); // return 2022-11-24T09:59:59Z + d.addSeconds(-60); + var dIsoS2 = d.toISOString(); // return 2022-11-24T09:58:59Z + d.addNanos(999999); + var dIsoN1 = d.toISOString(); // return 2022-11-24T09:58:59.000999999Z + d.addNanos(-1000000); + var dIsoN2 = d.toISOString(); // return 2022-11-24T09:58:58.999999999Z + return { + "dIsoY1": dIsoY1, + "dIsoY2": dIsoY2, + "dIsoY3": dIsoY3, + "dIsoM1": dIsoM1, + "dIsoM2": dIsoM2, + "dIsoM3": dIsoM3, + "dIsoW1": dIsoW1, + "dIsoW2": dIsoW2, + "dIsoD1": dIsoD1, + "dIsoD2": dIsoD2, + "dIsoD3": dIsoD3, + "dIsoH1": dIsoH1, + "dIsoH2": dIsoH2, + "dIsoMin1": dIsoMin1, + "dIsoMin2": dIsoMin2, + "dIsoS1": dIsoS1, + "dIsoS2": dIsoS2, + "dIsoN1": dIsoN1, + "dIsoN2": dIsoN2 + } + """, stringDateUTC); + LinkedHashMap expected = new LinkedHashMap<>(); + expected.put("dIsoY1", d.toISOString()); + d.addYears(1); + expected.put("dIsoY2", d.toISOString()); + d.addYears(-2); + expected.put("dIsoY3", d.toISOString()); + d.addMonths(2); + expected.put("dIsoM1", d.toISOString()); + d.addMonths(10); + expected.put("dIsoM2", d.toISOString()); + d.addMonths(-13); + expected.put("dIsoM3", d.toISOString()); + d.addWeeks(4); + expected.put("dIsoW1", d.toISOString()); + d.addWeeks(-5); + expected.put("dIsoW2", d.toISOString()); + d.addDays(6); + expected.put("dIsoD1", d.toISOString()); + d.addDays(45); + expected.put("dIsoD2", d.toISOString()); + d.addDays(-50); + expected.put("dIsoD3", d.toISOString()); + d.addHours(23); + expected.put("dIsoH1", d.toISOString()); + d.addHours(-47); + expected.put("dIsoH2", d.toISOString()); + d.addMinutes(59); + expected.put("dIsoMin1", d.toISOString()); + d.addMinutes(-60); + expected.put("dIsoMin2", d.toISOString()); + d.addSeconds(59); + expected.put("dIsoS1", d.toISOString()); + d.addSeconds(-60); + expected.put("dIsoS2", d.toISOString()); + d.addNanos(999999); + expected.put("dIsoN1", d.toISOString()); + d.addNanos(-1000000); + expected.put("dIsoN2", d.toISOString()); + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + } private List splice(List oldList, int start, int deleteCount, Object... values) { start = initStartIndex(oldList, start); diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java similarity index 71% rename from application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java rename to application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java index 28ea2d270a..6cf0ab9053 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java @@ -52,7 +52,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; @ExtendWith(MockitoExtension.class) -public class DefaultRuleEngineDeviceStateManagerTest { +public class DefaultDeviceStateManagerTest { @Mock private DeviceStateService deviceStateServiceMock; @@ -71,7 +71,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { @Captor private ArgumentCaptor queueCallbackCaptor; - private DefaultRuleEngineDeviceStateManager deviceStateManager; + private DefaultDeviceStateManager deviceStateManager; private static final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("57ab2e6c-bc4c-11ee-a506-0242ac120002")); private static final DeviceId DEVICE_ID = DeviceId.fromString("74a9053e-bc4c-11ee-a506-0242ac120002"); @@ -82,7 +82,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { @BeforeEach public void setup() { - deviceStateManager = new DefaultRuleEngineDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock); + deviceStateManager = new DefaultDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock); } @ParameterizedTest @@ -90,7 +90,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { "when onDeviceX() is called, then should route event to local service and call onSuccess() callback.") @MethodSource public void givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback( - BiConsumer onDeviceAction, Consumer actionVerification + BiConsumer onDeviceAction, Consumer actionVerification ) { // GIVEN given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); @@ -109,20 +109,24 @@ public class DefaultRuleEngineDeviceStateManagerTest { private static Stream givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback() { return Stream.of( Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) + ), + Arguments.of( + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS) ) ); } @@ -132,7 +136,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { "when onDeviceX() is called, then should route event to local service and call onFailure() callback.") @MethodSource public void givenRoutedToLocalAndProcessingFailure_whenOnDeviceAction_thenShouldCallLocalServiceAndFailureCallback( - Consumer exceptionThrowSetup, BiConsumer onDeviceAction, Consumer actionVerification + Consumer exceptionThrowSetup, BiConsumer onDeviceAction, Consumer actionVerification ) { // GIVEN given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); @@ -155,23 +159,28 @@ public class DefaultRuleEngineDeviceStateManagerTest { return Stream.of( Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) + ), + Arguments.of( + (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS) ) ); } @@ -181,7 +190,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { "when onDeviceX() is called, then should send correct queue message to external service with correct callback object.") @MethodSource public void givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback( - BiConsumer onDeviceAction, BiConsumer> actionVerification + BiConsumer onDeviceAction, BiConsumer> actionVerification ) { // WHEN ReflectionTestUtils.setField(deviceStateManager, "deviceStateService", Optional.empty()); @@ -203,7 +212,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { private static Stream givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback() { return Stream.of( Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -219,7 +228,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -235,7 +244,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -251,7 +260,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -265,6 +274,22 @@ public class DefaultRuleEngineDeviceStateManagerTest { .build(); then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture()); } + ), + Arguments.of( + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { + var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) + .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) + .setDeviceIdMSB(DEVICE_ID.getId().getMostSignificantBits()) + .setDeviceIdLSB(DEVICE_ID.getId().getLeastSignificantBits()) + .setInactivityTimeout(EVENT_TS) + .build(); + var toCoreMsg = TransportProtos.ToCoreMsg.newBuilder() + .setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg) + .build(); + then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture()); + } ) ); } diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 26c978bbd0..26e913eacc 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -38,9 +38,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger; import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.query.EntityData; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; @@ -72,7 +69,7 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; @@ -88,7 +85,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.thingsboard.server.service.state.DefaultDeviceStateService.ACTIVITY_STATE; import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_ALARM_TIME; -import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_TIMEOUT; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_ACTIVITY_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_CONNECT_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_DISCONNECT_TIME; @@ -211,7 +207,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastConnectTime) @@ -298,7 +294,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastDisconnectTime) @@ -421,13 +417,13 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(lastInactivityTime) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -465,12 +461,12 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -508,42 +504,6 @@ public class DefaultDeviceStateServiceTest { verify(service).fetchDeviceStateDataUsingSeparateRequests(deviceId); } - @Test - public void givenPersistToTelemetryAndDefaultInactivityTimeoutFetched_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() { - var defaultInactivityTimeoutInSec = 60L; - var latest = - Map.of( - EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000))), - EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L))) - ); - - process(latest, defaultInactivityTimeoutInSec); - } - - @Test - public void givenPersistToTelemetryAndNoInactivityTimeoutFetchedFromTimeSeries_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() { - var defaultInactivityTimeoutInSec = 60L; - var latest = - Map.of( - EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L))) - ); - - process(latest, defaultInactivityTimeoutInSec); - } - - private void process(Map> latest, long defaultInactivityTimeoutInSec) { - service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec); - service.setDefaultInactivityTimeoutMs(defaultInactivityTimeoutInSec * 1000); - service.setPersistToTelemetry(true); - - var deviceUuid = UUID.randomUUID(); - var deviceId = new DeviceId(deviceUuid); - - DeviceStateData deviceStateData = service.toDeviceStateData(new EntityData(deviceId, latest, Map.of()), new DeviceIdInfo(TenantId.SYS_TENANT_ID.getId(), UUID.randomUUID(), deviceUuid)); - - assertThat(deviceStateData.getState().getInactivityTimeout()).isEqualTo(5000L); - } - private void initStateService(long timeout) throws InterruptedException { service.stop(); reset(service, telemetrySubscriptionService); @@ -556,7 +516,7 @@ public class DefaultDeviceStateServiceTest { .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false)); PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of( new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi) - )); + ), Collections.emptyMap()); service.onApplicationEvent(event); Thread.sleep(100); } @@ -1002,7 +962,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.isActive()).isFalse(); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime) @@ -1114,7 +1074,7 @@ public class DefaultDeviceStateServiceTest { final long defaultTimeout = 1000; initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); - given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); + given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); TransportProtos.DeviceStateServiceMsgProto proto = TransportProtos.DeviceStateServiceMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) @@ -1170,7 +1130,7 @@ public class DefaultDeviceStateServiceTest { assertThat(attributeRequestCaptor.getAllValues()).hasSize(2) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { @@ -1179,7 +1139,7 @@ public class DefaultDeviceStateServiceTest { }); }) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { @@ -1196,7 +1156,7 @@ public class DefaultDeviceStateServiceTest { final long defaultTimeout = 1000; initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); - given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); + given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); long currentTime = System.currentTimeMillis(); DeviceState deviceState = DeviceState.builder() diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 7b4d6923f0..da3b214afb 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -43,6 +43,15 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; @@ -70,6 +79,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; @@ -79,6 +89,7 @@ import org.thingsboard.server.common.data.util.ThrowingRunnable; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -145,6 +156,8 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { protected TenantService tenantService; @Autowired protected EntityViewService entityViewService; + @Autowired + protected CalculatedFieldService calculatedFieldService; protected TenantId tenantId1; protected User tenantAdmin1; @@ -191,6 +204,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Asset asset = createAsset(tenantId1, null, assetProfile.getId(), "Asset 1"); DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), asset.getId()); Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId()) @@ -198,6 +212,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { try { return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() .exportCredentials(false) + .exportCalculatedFields(true) .build()); } catch (Exception e) { throw new RuntimeException(e); @@ -245,7 +260,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), any(), eq(ActionType.UPDATED), isNull()); - EntityExportData updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET); updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel()); Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity(); @@ -268,10 +282,27 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)); verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + // calculated field of imported device: + List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId()); + assertThat(calculatedFields.size()).isOne(); + var importedCalculatedField = calculatedFields.get(0); + assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + verify(tbClusterService).onCalculatedFieldUpdated(eq(importedCalculatedField), isNull(), any()); + EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); + + // update calculated field: + DeviceExportData deviceExportData = (DeviceExportData) getAndClone(entitiesExportData, EntityType.DEVICE); + deviceExportData.setCalculatedFields(deviceExportData.getCalculatedFields().stream().peek(field -> field.setName("t_" + field.getName())).toList()); + importEntity(tenantAdmin2, deviceExportData).getSavedEntity(); + + calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId()); + assertThat(calculatedFields.size()).isOne(); + importedCalculatedField = calculatedFields.get(0); + assertThat(importedCalculatedField.getName()).startsWith("t_"); } @Test @@ -290,12 +321,15 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1"); EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1"); + CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), device.getId()); + Map ids = new HashMap<>(); for (EntityId entityId : List.of(customer.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), asset.getId(), deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() .saveCredentials(false) + .saveCalculatedFields(true) .build()); ids.put(entityId, (EntityId) importResult.getSavedEntity().getId()); } @@ -325,10 +359,16 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId()); - Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity(); + EntityExportData entityExportData = exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())); + Device exportedDevice = entityExportData.getEntity(); assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId()); + List calculatedFields = ((DeviceExportData) entityExportData).getCalculatedFields(); + assertThat(calculatedFields.size()).isOne(); + CalculatedField field = calculatedFields.get(0); + assertThat(field.getName()).isEqualTo(calculatedField.getName()); + EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); @@ -340,7 +380,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { deviceProfileService.saveDeviceProfile(importedDeviceProfile); } - protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { Device device = new Device(); device.setTenantId(tenantId); @@ -549,9 +588,43 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { return relation; } + private CalculatedField createCalculatedField(TenantId tenantId, EntityId entityId, EntityId referencedEntityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setVersion(1L); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId) throws Exception { return exportEntity(user, entityId, EntityExportSettings.builder() .exportCredentials(true) + .exportCalculatedFields(true) .build()); } @@ -562,6 +635,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData) throws Exception { return importEntity(user, exportData, EntityImportSettings.builder() .saveCredentials(true) + .saveCalculatedFields(true) .build()); } diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index 0088ae2b7b..06f61ca4e5 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -44,6 +44,15 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; @@ -562,6 +571,81 @@ public class VersionControlTest extends AbstractControllerTest { checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, importedDeviceProfile); } + @Test + public void testVcWithCalculatedFields_betweenTenants() throws Exception { + Asset asset = createAsset(null, null, "Asset 1"); + Device device = createDevice(null, null, "Device 1", "test1"); + CalculatedField calculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); + String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + loginTenant2(); + loadVersion(versionId, config -> { + config.setLoadCredentials(false); + }, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + Asset importedAsset = findAsset(asset.getName()); + Device importedDevice = findDevice(device.getName()); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId()); + assertThat(importedCalculatedFields).size().isOne(); + assertThat(importedCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(calculatedField.getId()); + }); + } + + @Test + public void testVcWithReferencedCalculatedFields_betweenTenants() throws Exception { + Asset asset = createAsset(null, null, "Asset 1"); + Device device = createDevice(null, null, "Device 1", "test1"); + CalculatedField deviceCalculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); + CalculatedField assetCalculatedField = createCalculatedField("CalculatedField2", asset.getId(), device.getId()); + String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + loginTenant2(); + loadVersion(versionId, config -> { + config.setLoadCredentials(false); + }, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + Asset importedAsset = findAsset(asset.getName()); + Device importedDevice = findDevice(device.getName()); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedDeviceCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId()); + assertThat(importedDeviceCalculatedFields).size().isOne(); + assertThat(importedDeviceCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(deviceCalculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(deviceCalculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(deviceCalculatedField.getId()); + }); + + List importedAssetCalculatedFields = findCalculatedFieldsByEntityId(importedAsset.getId()); + assertThat(importedAssetCalculatedFields).size().isOne(); + assertThat(importedAssetCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(assetCalculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(assetCalculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(assetCalculatedField.getId()); + }); + } + + @Test + public void testVcWithCalculatedFields_sameTenant() throws Exception { + Asset asset = createAsset(null, null, "Asset 1"); + CalculatedField calculatedField = createCalculatedField("CalculatedField", asset.getId(), asset.getId()); + String versionId = createVersion("asset and field", EntityType.ASSET); + + loadVersion(versionId, EntityType.ASSET); + CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(asset.getId()); + assertThat(importedCalculatedField.getId()).isEqualTo(calculatedField.getId()); + assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(importedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(importedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + } + private & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); @@ -626,10 +710,10 @@ public class VersionControlTest extends AbstractControllerTest { request.setEntityTypes(Arrays.stream(entityTypes).collect(Collectors.toMap(t -> t, entityType -> { EntityTypeVersionCreateConfig config = new EntityTypeVersionCreateConfig(); config.setAllEntities(true); - config.setSaveRelations(true); config.setSaveAttributes(true); config.setSaveCredentials(true); + config.setSaveCalculatedFields(true); return config; }))); @@ -695,6 +779,7 @@ public class VersionControlTest extends AbstractControllerTest { config.setLoadAttributes(true); config.setLoadRelations(true); config.setLoadCredentials(true); + config.setLoadCalculatedFields(true); config.setRemoveOtherEntities(false); config.setFindExistingEntityByName(true); configModifier.accept(config); @@ -941,6 +1026,38 @@ public class VersionControlTest extends AbstractControllerTest { return doPost("/api/v2/relation", relation, EntityRelation.class); } + private CalculatedField createCalculatedField(String name, EntityId entityId, EntityId referencedEntityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setVersion(1L); + return doPost("/api/calculatedField", calculatedField, CalculatedField.class); + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); @@ -995,11 +1112,18 @@ public class VersionControlTest extends AbstractControllerTest { private RuleChain findRuleChain(String name) throws Exception { return doGetTypedWithPageLink("/api/ruleChains?", new TypeReference>() {}, new PageLink(100, 0, name)).getData().get(0); - } private RuleChainMetaData findRuleChainMetaData(RuleChainId ruleChainId) throws Exception { return doGet("/api/ruleChain/" + ruleChainId + "/metadata", RuleChainMetaData.class); } + private CalculatedField findCalculatedFieldByEntityId(EntityId entityId) throws Exception { + return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference>() {}, new PageLink(100, 0)).getData().get(0); + } + + private List findCalculatedFieldsByEntityId(EntityId entityId) throws Exception { + return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference>() {}, new PageLink(100, 0)).getData(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index b8bfa13e50..6bb4f0e2fe 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -74,6 +75,7 @@ import java.util.stream.Stream; import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; @@ -159,7 +161,7 @@ class DefaultTelemetrySubscriptionServiceTest { }).when(calculatedFieldQueueService).pushRequestToQueue(any(TimeseriesSaveRequest.class), any(), any()); // send partition change event so currentPartitions set is populated - telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)))); + telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)), Collections.emptyMap())); } @AfterEach @@ -168,6 +170,28 @@ class DefaultTelemetrySubscriptionServiceTest { tsCallBackExecutor.shutdownNow(); } + /* --- Save time series API --- */ + + @Test + void shouldThrowErrorWhenTryingToSaveTimeseriesForApiUsageState() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(new ApiUsageStateId(UUID.randomUUID())) + .entries(sampleTelemetry) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .build(); + + // WHEN + assertThatThrownBy(() -> telemetryService.saveTimeseries(request)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Can't update API Usage State!"); + + // THEN + then(tsService).shouldHaveNoInteractions(); + } + @Test void shouldReportStorageDataPointsApiUsageWhenTimeSeriesIsSaved() { // GIVEN @@ -377,7 +401,7 @@ class DefaultTelemetrySubscriptionServiceTest { ); } - // used to emulate versions returned by save latest API + // used to emulate sequence numbers returned by save latest API private static List listOfNNumbers(int N) { return LongStream.range(0, N).boxed().toList(); } diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 3022650d90..b8bdcf67e6 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -58,3 +58,6 @@ sql.ttl.edge_events.edge_event_ttl=2592000 server.log_controller_error_stack_trace=false transport.gateway.dashboard.sync.enabled=false + +queue.edqs.sync.enabled=false +queue.edqs.api.supported=false diff --git a/application/src/test/resources/update/330/device_profile_001_out.json b/application/src/test/resources/update/330/device_profile_001_out.json index 9a349c6638..29e2241ee9 100644 --- a/application/src/test/resources/update/330/device_profile_001_out.json +++ b/application/src/test/resources/update/330/device_profile_001_out.json @@ -64,7 +64,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } @@ -103,7 +104,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index abde4cce97..609a61d575 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -26,6 +26,8 @@ public interface TbQueueRequestTemplate send(Request request, long timeoutNs); + ListenableFuture send(Request request, Integer partition); + void stop(); void setMessagesStats(MessagesStats messagesStats); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java index e7e5361381..918d656af0 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -15,9 +15,17 @@ */ package org.thingsboard.server.queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + public interface TbQueueResponseTemplate { - void init(TbQueueHandler handler); + void subscribe(); + + void subscribe(Set partitions); + + void launch(TbQueueHandler handler); void stop(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java new file mode 100644 index 0000000000..51c631fe4f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +public enum ObjectType { + TENANT, + TENANT_PROFILE, + CUSTOMER, + QUEUE, + RPC, + RULE_CHAIN, + OTA_PACKAGE, + RESOURCE, + EVENT, + RULE_NODE, + USER, + EDGE, + WIDGETS_BUNDLE, + WIDGET_TYPE, + DASHBOARD, + DEVICE_PROFILE, + DEVICE, + DEVICE_CREDENTIALS, + ASSET_PROFILE, + ASSET, + ENTITY_VIEW, + ALARM, + ENTITY_ALARM, + OAUTH2_CLIENT, + OAUTH2_DOMAIN, + OAUTH2_MOBILE, + USER_SETTINGS, + NOTIFICATION_TARGET, + NOTIFICATION_TEMPLATE, + NOTIFICATION_RULE, + ALARM_COMMENT, + API_USAGE_STATE, + QUEUE_STATS, + + AUDIT_LOG, + RELATION, + ATTRIBUTE_KV, + LATEST_TS_KV; + + public static final Set edqsTenantTypes = EnumSet.of( + TENANT, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS + ); + public static final Set edqsTypes = EnumSet.copyOf(edqsTenantTypes); + public static final Set edqsSystemTypes = EnumSet.of(TENANT, USER, DASHBOARD, + API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); + public static final Set unversionedTypes = EnumSet.of( + QUEUE_STATS // created once, never updated + ); + + static { + edqsTypes.addAll(List.of(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + } + + public EntityType toEntityType() { + return EntityType.valueOf(name()); + } + + public static ObjectType fromEntityType(EntityType entityType) { + try { + return ObjectType.valueOf(entityType.name()); + } catch (Exception e) { + return null; + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java index d1b4a35b7b..22934de813 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java @@ -19,7 +19,6 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index b7c7584931..5c7f0bb53d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -58,6 +59,7 @@ public class TenantProfile extends BaseData implements HasName @Schema(description = "If enabled, will push all messages related to this tenant and processed by the rule engine into separate queue. " + "Useful for complex microservices deployments, to isolate processing of the data for specific tenants", example = "false") private boolean isolatedTbRuleEngine; + @Valid @Schema(description = "Complex JSON object that contains profile settings: queue configs, max devices, max assets, rate limits, etc.") private transient TenantProfileData profileData; @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index d4aa0b60a3..b86f30ca78 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -23,7 +23,6 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -37,11 +36,14 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serial; + @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity, HasDebugSettings { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, HasDebugSettings { + @Serial private static final long serialVersionUID = 4491966747773381420L; private TenantId tenantId; @@ -66,9 +68,6 @@ public class CalculatedField extends BaseData implements HasN @Getter @Setter private Long version; - @Getter - @Setter - private CalculatedFieldId externalId; public CalculatedField() { super(); @@ -78,7 +77,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; @@ -86,7 +85,19 @@ public class CalculatedField extends BaseData implements HasN this.configurationVersion = configurationVersion; this.configuration = configuration; this.version = version; - this.externalId = externalId; + } + + public CalculatedField(CalculatedField calculatedField) { + super(calculatedField); + this.tenantId = calculatedField.tenantId; + this.entityId = calculatedField.entityId; + this.type = calculatedField.type; + this.name = calculatedField.name; + this.debugMode = calculatedField.debugMode; + this.debugSettings = calculatedField.debugSettings; + this.configurationVersion = calculatedField.configurationVersion; + this.configuration = calculatedField.configuration; + this.version = calculatedField.version; } @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") @@ -112,7 +123,6 @@ public class CalculatedField extends BaseData implements HasN .append(", configurationVersion=").append(configurationVersion) .append(", configuration=").append(configuration) .append(", version=").append(version) - .append(", externalId=").append(externalId) .append(", createdTime=").append(createdTime) .append(", id=").append(id).append(']') .toString(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 49e393b19f..f2b4948837 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -26,5 +26,6 @@ public class Output { private String name; private OutputType type; private AttributeScope scope; + private Integer decimalsByDefault; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java new file mode 100644 index 0000000000..c162365257 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class AttributeKv implements EdqsObject { + + private EntityId entityId; + private AttributeScope scope; + private String key; + private Long version; + + private DataPoint dataPoint; // optional (on deletion) + + private Long lastUpdateTs; // only for serialization + private KvEntry value; // only for serialization + + public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = attributeKvEntry.getKey(); + this.version = version; + this.lastUpdateTs = attributeKvEntry.getLastUpdateTs(); + this.value = attributeKvEntry; + } + + public AttributeKv(EntityId entityId, AttributeScope scope, String key, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = key; + this.version = version; + } + + @Override + public String key() { + return "a_" + entityId + "_" + scope + "_" + key; + } + + @Override + public Long version() { + return version; + } + + @Override + public ObjectType type() { + return ObjectType.ATTRIBUTE_KV; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java new file mode 100644 index 0000000000..a6f30c8004 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import org.thingsboard.server.common.data.kv.DataType; + +public interface DataPoint { + + String NOT_SUPPORTED = "Not supported!"; + + long getTs(); + + DataType getType(); + + String getStr(); + + long getLong(); + + double getDouble(); + + boolean getBool(); + + String getJson(); + + String valueToString(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java new file mode 100644 index 0000000000..50c8d268de --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@AllArgsConstructor +@Builder +public class EdqsEvent { + + private final TenantId tenantId; + private final ObjectType objectType; + private final EdqsEventType eventType; + private final EdqsObject object; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java new file mode 100644 index 0000000000..df31048365 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +public enum EdqsEventType { + UPDATED, + DELETED +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java new file mode 100644 index 0000000000..a74c90208a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.thingsboard.server.common.data.ObjectType; + +public interface EdqsObject { + + @JsonIgnore + String key(); + + @JsonIgnore + Long version(); + + @JsonIgnore + ObjectType type(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java new file mode 100644 index 0000000000..12f7068c71 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +@Data +@JsonIgnoreProperties +public class EdqsSyncRequest { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java new file mode 100644 index 0000000000..c22ef147e3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.EntityIdFields; + +import java.util.UUID; + +@Data +@NoArgsConstructor +public class Entity implements EdqsObject { + + private EntityType type; + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + private EntityFields fields; + + public Entity(EntityType type) { + this.type = type; + } + + public Entity(EntityType type, EntityFields fields) { + this.type = type; + this.fields = fields; + } + + public Entity(EntityType entityType, UUID id, long version) { + this.type = entityType; + this.fields = new EntityIdFields(id, version); + } + + @Override + public String key() { + return "e_" + fields.getId().toString(); + } + + @Override + public Long version() { + return fields.getVersion(); + } + + @Override + public ObjectType type() { + return ObjectType.fromEntityType(type); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java new file mode 100644 index 0000000000..8bd69c41a4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class LatestTsKv implements EdqsObject { + + private EntityId entityId; + private String key; + private Long version; + + private DataPoint dataPoint; // optional (on deletion) + + private Long ts; // only for serialization + private KvEntry value; // only for serialization + + public LatestTsKv(EntityId entityId, TsKvEntry tsKvEntry, Long version) { + this.entityId = entityId; + this.key = tsKvEntry.getKey(); + this.ts = tsKvEntry.getTs(); + this.version = version != null ? version : 0L; + this.value = tsKvEntry; + } + + public LatestTsKv(EntityId entityId, String key, Long version) { + this.entityId = entityId; + this.key = key; + this.version = version != null ? version : 0L; + } + + public String key() { + return "l_" + entityId + "_" + key; + } + + @Override + public Long version() { + return version; + } + + @Override + public ObjectType type() { + return ObjectType.LATEST_TS_KV; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java new file mode 100644 index 0000000000..78bebba20a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class ToCoreEdqsMsg { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java new file mode 100644 index 0000000000..c4f262fbf0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ToCoreEdqsRequest { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + + @JsonIgnore + public ToCoreEdqsMsg toInternalMsg() { + return new ToCoreEdqsMsg(syncRequest, apiEnabled); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java new file mode 100644 index 0000000000..4aad5eb4dd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.CustomerId; + +import java.util.UUID; + +@Data +@SuperBuilder +public class AbstractEntityFields implements EntityFields { + + private UUID id; + private long createdTime; + private UUID tenantId; + private UUID customerId; + private String name; + private Long version; + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { + this.id = id; + this.createdTime = createdTime; + this.tenantId = tenantId; + this.customerId = (customerId != null && customerId != CustomerId.NULL_UUID) ? customerId : null; + this.name = name; + this.version = version; + } + + public AbstractEntityFields() { + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + this(id, createdTime, tenantId, null, name, version); + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, Long version) { + this(id, createdTime, tenantId, customerId, null, version); + + } + + public AbstractEntityFields(UUID id, long createdTime, String name, Long version) { + this(id, createdTime, null, name, version); + } + + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId) { + this(id, createdTime, tenantId, null, null, null); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java new file mode 100644 index 0000000000..d10f375bc1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@SuperBuilder +public class ApiUsageStateFields extends AbstractEntityFields { + + private EntityId entityId; + private ApiUsageStateValue transportState; + private ApiUsageStateValue dbStorageState; + private ApiUsageStateValue reExecState; + private ApiUsageStateValue jsExecState; + private ApiUsageStateValue tbelExecState; + private ApiUsageStateValue emailExecState; + private ApiUsageStateValue smsExecState; + private ApiUsageStateValue alarmExecState; + + public ApiUsageStateFields(UUID id, long createdTime, UUID tenantId, UUID entityId, String entityType, ApiUsageStateValue transportState, ApiUsageStateValue dbStorageState, + ApiUsageStateValue reExecState, ApiUsageStateValue jsExecState, ApiUsageStateValue tbelExecState, + ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState, + Long version) { + super(id, createdTime, tenantId, null, null, version); + this.entityId = (entityType != null && entityId != null) ? EntityIdFactory.getByTypeAndUuid(entityType, entityId) : null; + this.transportState = transportState; + this.dbStorageState = dbStorageState; + this.reExecState = reExecState; + this.jsExecState = jsExecState; + this.tbelExecState = tbelExecState; + this.emailExecState = emailExecState; + this.smsExecState = smsExecState; + this.alarmExecState = alarmExecState; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java new file mode 100644 index 0000000000..b4020e5788 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetFields extends AbstractEntityFields implements ProfileAwareFields { + + private String type; + private UUID assetProfileId; + private String label; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return assetProfileId; + } + + public AssetFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, + Long version, String type, String label, UUID assetProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.assetProfileId = assetProfileId; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java new file mode 100644 index 0000000000..56d8691cbe --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public AssetProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java new file mode 100644 index 0000000000..4132e9e094 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class CustomerFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + + public CustomerFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo, + String country, String state, String city, String address, String address2, String zip, String phone, String email) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java new file mode 100644 index 0000000000..0061b7e10b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + + +@Data +@NoArgsConstructor +@SuperBuilder +public class DashboardFields extends AbstractEntityFields { + + private static ObjectMapper objectMapper = new ObjectMapper(); + private List assignedCustomerIds; + + public DashboardFields(UUID id, long createdTime, UUID tenantId, String assignedCustomers, String name, Long version) { + super(id, createdTime, tenantId, name, version); + this.assignedCustomerIds = getCustomerIds(assignedCustomers); + } + + private static List getCustomerIds(String assignedCustomers) { + List ids = new ArrayList<>(); + if (assignedCustomers == null || assignedCustomers.isEmpty()) { + return ids; + } + try { + JsonNode rootNode = objectMapper.readTree(assignedCustomers); + for (JsonNode node : rootNode) { + String idStr = node.path("customerId").path("id").asText(); + if (!idStr.isEmpty()) { + ids.add(UUID.fromString(idStr)); + } + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return ids; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java new file mode 100644 index 0000000000..ea1ef383de --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceFields extends AbstractEntityFields implements ProfileAwareFields { + + private String label; + private String type; + private UUID deviceProfileId; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return deviceProfileId; + } + + public DeviceFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, String type, + String label, UUID deviceProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.label = label; + this.type = type; + this.deviceProfileId = deviceProfileId; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java new file mode 100644 index 0000000000..1e8d56ab14 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.DeviceProfileType; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceProfileFields extends AbstractEntityFields { + + private String type; + private boolean isDefault; + + public DeviceProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, DeviceProfileType type, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.type = type.name(); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java new file mode 100644 index 0000000000..483a7f2680 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EdgeFields extends AbstractEntityFields { + + private String type; + private String label; + private String additionalInfo; + + public EdgeFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, + String type, String label, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java new file mode 100644 index 0000000000..532c4a92ac --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -0,0 +1,179 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonInclude; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public interface EntityFields { + + Logger log = LoggerFactory.getLogger(EntityFields.class); + + default UUID getId() { + return null; + } + + default UUID getTenantId() { + return null; + } + + default UUID getCustomerId() { + return null; + } + + default List getAssignedCustomerIds() { + return Collections.emptyList(); + } + + default long getCreatedTime() { + return 0; + } + + default String getName() { + return ""; + } + + default String getType() { + return ""; + } + + default String getLabel() { + return ""; + } + + default String getAdditionalInfo() { + return ""; + } + + default String getEmail() { + return ""; + } + + default String getCountry() { + return ""; + } + + default String getState() { + return ""; + } + + default String getCity() { + return ""; + } + + default String getAddress() { + return ""; + } + + default String getAddress2() { + return ""; + } + + default String getZip() { + return ""; + } + + default String getPhone() { + return ""; + } + + default String getRegion() { + return ""; + } + + default String getFirstName() { + return ""; + } + + default String getLastName() { + return ""; + } + + default boolean isEdgeTemplate() { + return false; + } + + default String getConfiguration() { + return ""; + } + + default String getSchedule() { + return ""; + } + + default EntityId getOriginatorId() { + return null; + } + + default String getQueueName() { + return ""; + } + + default String getServiceId() { + return ""; + } + + default boolean isDefault() { + return false; + } + + default UUID getOwnerId() { + return null; + } + + default Long getVersion() { + return null; + } + + default String getAsString(String key) { + return switch (key) { + case "createdTime" -> Long.toString(getCreatedTime()); + case "type" -> getType(); + case "label" -> getLabel(); + case "additionalInfo" -> getAdditionalInfo(); + case "email" -> getEmail(); + case "country" -> getCountry(); + case "state" -> getState(); + case "city" -> getCity(); + case "address" -> getAddress(); + case "address2" -> getAddress2(); + case "zip" -> getZip(); + case "phone" -> getPhone(); + case "region" -> getRegion(); + case "firstName" -> getFirstName(); + case "lastName" -> getLastName(); + case "edgeTemplate" -> Boolean.toString(isEdgeTemplate()); + case "configuration" -> getConfiguration(); + case "schedule" -> getSchedule(); + case "originatorId" -> getOriginatorId().getId().toString(); + case "originatorType" -> getOriginatorId().getEntityType().toString(); + case "queueName" -> getQueueName(); + case "serviceId" -> getServiceId(); + default -> { + log.warn("Unknown field '{}'", key); + yield null; + } + }; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java new file mode 100644 index 0000000000..835577c4f1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityIdFields implements EntityFields { + + private UUID id; + private Long version; + + public EntityIdFields(UUID id, Long version) { + this.id = id; + this.version = version; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java new file mode 100644 index 0000000000..7674fba200 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityViewFields extends AbstractEntityFields { + + private String type; + private String additionalInfo; + + public EntityViewFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, String type, String additionalInfo, Long version) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.additionalInfo = additionalInfo; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java new file mode 100644 index 0000000000..a36514248c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -0,0 +1,299 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.widget.WidgetType; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.util.UUID; + +public class FieldsUtil { + + public static EntityFields toFields(Object entity) { + if (entity instanceof Customer customer) { + return toFields(customer); + } else if (entity instanceof Tenant tenant) { + return toFields(tenant); + } else if (entity instanceof TenantProfile tenantProfile) { + return toFields(tenantProfile); + } else if (entity instanceof Device device) { + return toFields(device); + } else if (entity instanceof Asset asset) { + return toFields(asset); + } else if (entity instanceof Edge edge) { + return toFields(edge); + } else if (entity instanceof EntityView entityView) { + return toFields(entityView); + } else if (entity instanceof User user) { + return toFields(user); + } else if (entity instanceof Dashboard dashboard) { + return toFields(dashboard); + } else if (entity instanceof RuleChain ruleChain) { + return toFields(ruleChain); + } else if (entity instanceof RuleNode ruleNode) { + return toFields(ruleNode); + } else if (entity instanceof WidgetType widgetType) { + return toFields(widgetType); + } else if (entity instanceof WidgetsBundle widgetsBundle) { + return toFields(widgetsBundle); + } else if (entity instanceof DeviceProfile deviceProfile) { + return toFields(deviceProfile); + } else if (entity instanceof AssetProfile assetProfile) { + return toFields(assetProfile); + } else if (entity instanceof QueueStats queueStats) { + return toFields(queueStats); + } else if (entity instanceof ApiUsageState apiUsageState) { + return toFields(apiUsageState); + } else { + throw new IllegalArgumentException("Unsupported entity type: " + entity.getClass().getName()); + } + } + + private static CustomerFields toFields(Customer entity) { + return CustomerFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .version(entity.getVersion()) + .build(); + } + + private static TenantFields toFields(Tenant entity) { + return TenantFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .region(entity.getRegion()) + .version(entity.getVersion()) + .build(); + } + + private static TenantProfileFields toFields(TenantProfile tenantProfile) { + return TenantProfileFields.builder() + .id(tenantProfile.getUuidId()) + .createdTime(tenantProfile.getCreatedTime()) + .name(tenantProfile.getName()) + .isDefault(tenantProfile.isDefault()) + .build(); + } + + private static DeviceFields toFields(Device entity) { + return DeviceFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .deviceProfileId(entity.getDeviceProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static AssetFields toFields(Asset entity) { + return AssetFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .assetProfileId(entity.getAssetProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EdgeFields toFields(Edge entity) { + return EdgeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EntityViewFields toFields(EntityView entity) { + return EntityViewFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static UserFields toFields(User entity) { + return UserFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .firstName(entity.getFirstName()) + .lastName(entity.getLastName()) + .email(entity.getEmail()) + .phone(entity.getPhone()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static DashboardFields toFields(Dashboard entity) { + return DashboardFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .version(entity.getVersion()) + .build(); + } + + private static RuleChainFields toFields(RuleChain entity) { + return RuleChainFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static RuleNodeFields toFields(RuleNode entity) { + return RuleNodeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .build(); + } + + private static WidgetTypeFields toFields(WidgetType entity) { + return WidgetTypeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static WidgetsBundleFields toFields(WidgetsBundle entity) { + return WidgetsBundleFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static AssetProfileFields toFields(AssetProfile entity) { + return AssetProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static DeviceProfileFields toFields(DeviceProfile entity) { + return DeviceProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .type(DeviceProfileType.DEFAULT.name()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static QueueStatsFields toFields(QueueStats entity) { + return QueueStatsFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .queueName(entity.getQueueName()) + .serviceId(entity.getServiceId()) + .build(); + } + + private static ApiUsageStateFields toFields(ApiUsageState entity) { + return ApiUsageStateFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(entity.getEntityId().getEntityType() == EntityType.CUSTOMER ? entity.getEntityId().getId() : null) + .entityId(entity.getEntityId()) + .transportState(entity.getTransportState()) + .dbStorageState(entity.getDbStorageState()) + .reExecState(entity.getReExecState()) + .jsExecState(entity.getJsExecState()) + .tbelExecState(entity.getTbelExecState()) + .emailExecState(entity.getEmailExecState()) + .smsExecState(entity.getSmsExecState()) + .alarmExecState(entity.getAlarmExecState()) + .version(entity.getVersion()) + .build(); + } + + public static String getText(JsonNode node) { + return node != null ? node.asText() : ""; + } + + private static UUID getCustomerId(CustomerId customerId) { + return (customerId != null && !customerId.getId().equals(CustomerId.NULL_UUID)) ? customerId.getId() : null; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java new file mode 100644 index 0000000000..68366a6b4d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; + +import java.util.UUID; + +@NoArgsConstructor +public class GenericFields extends AbstractEntityFields { + + public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java new file mode 100644 index 0000000000..4228755808 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import java.util.UUID; + +public interface ProfileAwareFields extends EntityFields { + + String getProfileName(); + + UUID getProfileId(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java new file mode 100644 index 0000000000..cbc1ad4b8e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class QueueStatsFields extends AbstractEntityFields { + + private String queueName; + private String serviceId; + + @Override + public String getName() { + return queueName + '_' + serviceId; + } + + public QueueStatsFields(UUID id, long createdTime, UUID tenantId, String queueName, String serviceId) { + super(id, createdTime, tenantId); + this.queueName = queueName; + this.serviceId = serviceId; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java new file mode 100644 index 0000000000..a047eebd85 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleChainFields extends AbstractEntityFields { + + private String additionalInfo; + + public RuleChainFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java new file mode 100644 index 0000000000..8aa5b7f42f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleNodeFields implements EntityFields { + + private UUID id; + private long createdTime; + private String name; + private String additionalInfo; + + public RuleNodeFields(UUID id, long createdTime, String name, JsonNode additionalInfo) { + this.id = id; + this.createdTime = createdTime; + this.name = name; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java new file mode 100644 index 0000000000..342e2974a4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + private String region; + + public TenantFields(UUID id, long createdTime, String name, Long version, + JsonNode additionalInfo, String country, String state, String city, String address, + String address2, String zip, String phone, String email, String region) { + super(id, createdTime, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + this.region = region; + } + + @Override + public UUID getTenantId() { + return getId(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java new file mode 100644 index 0000000000..b897bd1334 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public TenantProfileFields(UUID id, long createdTime, String name, boolean isDefault) { + super(id, createdTime, TenantId.SYS_TENANT_ID.getId(), null, name, 0L); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java new file mode 100644 index 0000000000..9863506ed4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class UserFields extends AbstractEntityFields { + + private String firstName; + private String lastName; + private String email; + private String phone; + private String additionalInfo; + + public UserFields(UUID id, long createdTime, UUID tenantId, UUID customerId, + Long version, String firstName, String lastName, String email, + String phone, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, version); + this.firstName = firstName; + this.lastName = lastName; + this.email = email; + this.phone = phone; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java new file mode 100644 index 0000000000..4fd0079012 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetTypeFields extends AbstractEntityFields { + + public WidgetTypeFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java new file mode 100644 index 0000000000..f2fb1df508 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetsBundleFields extends AbstractEntityFields { + + public WidgetsBundleFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java new file mode 100644 index 0000000000..e7d8b1df49 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EdqsRequest { + + private EntityDataQuery entityDataQuery; + private EntityCountQuery entityCountQuery; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java new file mode 100644 index 0000000000..d4e53dea9f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class EdqsResponse { + + private PageData entityDataQueryResult; + private Long entityCountQueryResult; + private String error; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java new file mode 100644 index 0000000000..67a05323a1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; + +import java.util.Collections; +import java.util.Map; + +@Data +@RequiredArgsConstructor +public class QueryResult { + + private final EntityId entityId; + private final Map> latest; + + public EntityData toOldEntityData() { + return new EntityData(entityId, latest, Collections.emptyMap(), Collections.emptyMap()); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java index 0a83f1a19f..e17a066d88 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java @@ -20,11 +20,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; import org.thingsboard.server.common.data.EntityType; +import java.io.Serial; import java.util.UUID; @Schema public class CalculatedFieldId extends UUIDBased implements EntityId { + @Serial private static final long serialVersionUID = 1L; @JsonCreator @@ -41,4 +43,5 @@ public class CalculatedFieldId extends UUIDBased implements EntityId { public EntityType getEntityType() { return EntityType.CALCULATED_FIELD; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java index f17151dea9..a57ee7ee48 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java @@ -15,11 +15,15 @@ */ package org.thingsboard.server.common.data.id; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.UUID; public class UserAuthSettingsId extends UUIDBased { - public UserAuthSettingsId(UUID id) { + @JsonCreator + public UserAuthSettingsId(@JsonProperty("id") UUID id) { super(id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java similarity index 72% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java rename to common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java index bb5fdddb9d..ace36a1f37 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.query; +package org.thingsboard.server.common.data.permission; import lombok.AllArgsConstructor; import lombok.Getter; @@ -21,8 +21,12 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + @AllArgsConstructor -public class QuerySecurityContext { +public class QueryContext { @Getter private final TenantId tenantId; @@ -33,7 +37,14 @@ public class QuerySecurityContext { @Getter private final boolean ignorePermissionCheck; - public QuerySecurityContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { + @Getter + private final Map relatedParentIdMap = new HashMap<>(); + + public QueryContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { this(tenantId, customerId, entityType, false); } + + public boolean isTenantUser() { + return customerId == null || customerId.isNullUid(); + } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java index 9ae56a1d60..8c4ae1e8ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.common.data.query; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.validation.NoXss; @@ -26,7 +25,6 @@ import java.io.Serializable; @RequiredArgsConstructor public class DynamicValue implements Serializable { - @JsonIgnore private T resolvedValue; private final DynamicValueSourceType sourceType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index e2e7e18d9f..12b5331651 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.query; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.ToString; @@ -24,6 +25,7 @@ import java.util.List; @Schema @ToString +@JsonIgnoreProperties(ignoreUnknown = true) public class EntityCountQuery { @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java index 1519e3910d..328882bbd0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java @@ -16,20 +16,22 @@ package org.thingsboard.server.common.data.query; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; -import lombok.RequiredArgsConstructor; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; import java.util.Map; @Data -@RequiredArgsConstructor +@AllArgsConstructor +@NoArgsConstructor public class EntityData { - private final EntityId entityId; - private final Map> latest; - private final Map timeseries; - private final Map aggLatest; + private EntityId entityId; + private Map> latest; + private Map timeseries; + private Map aggLatest; public EntityData(EntityId entityId, Map> latest, Map timeseries) { this(entityId, latest, timeseries, null); @@ -44,4 +46,5 @@ public class EntityData { aggLatest.clear(); } } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java index 4be9633d96..5507f53f08 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java @@ -39,7 +39,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), @JsonSubTypes.Type(value = EntityViewSearchQueryFilter.class, name = "entityViewSearchQuery"), - @JsonSubTypes.Type(value = EdgeSearchQueryFilter.class, name = "edgeSearchQuery")}) + @JsonSubTypes.Type(value = EdgeSearchQueryFilter.class, name = "edgeSearchQuery") +}) public interface EntityFilter { @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java index 5b8c86d7d1..ca63d34a84 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java @@ -13,21 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.common.data.queue; public enum ProcessingStrategyType { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 7cbedcbacb..8980d0e634 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -25,6 +25,8 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.validation.Length; @@ -34,7 +36,7 @@ import java.io.Serializable; @Schema @EqualsAndHashCode(exclude = "additionalInfoBytes") @ToString(exclude = {"additionalInfoBytes"}) -public class EntityRelation implements HasVersion, Serializable { +public class EntityRelation implements HasVersion, Serializable, EdqsObject { private static final long serialVersionUID = 2807343040519543363L; @@ -107,7 +109,7 @@ public class EntityRelation implements HasVersion, Serializable { return typeGroup; } - @Schema(description = "Additional parameters of the relation",implementation = com.fasterxml.jackson.databind.JsonNode.class) + @Schema(description = "Additional parameters of the relation", implementation = com.fasterxml.jackson.databind.JsonNode.class) public JsonNode getAdditionalInfo() { return BaseDataWithAdditionalInfo.getJson(() -> additionalInfo, () -> additionalInfoBytes); } @@ -116,4 +118,19 @@ public class EntityRelation implements HasVersion, Serializable { BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes); } + @JsonIgnore + public String key() { + return "r_" + from + "_" + to + "_" + typeGroup + "_" + type; + } + + @Override + public Long version() { + return version; + } + + @Override + public ObjectType type() { + return ObjectType.RELATION; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index bfa82caa2b..98948ca642 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -38,4 +38,5 @@ public class DeviceExportData extends EntityExportData { public boolean hasCredentials() { return credentials != null; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 7bc9a63c29..072be6acf2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; @@ -55,6 +56,8 @@ public class EntityExportData> { public static final Comparator attrComparator = Comparator .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + public static final Comparator calculatedFieldsComparator = Comparator.comparing(CalculatedField::getName); + @JsonProperty(index = 2) @JsonTbEntity private E entity; @@ -65,6 +68,9 @@ public class EntityExportData> { private List relations; @JsonProperty(index = 101) private Map> attributes; + @JsonProperty(index = 102) + @JsonIgnoreProperties({"id", "entityId", "createdTime", "version"}) + private List calculatedFields; public EntityExportData sort() { if (relations != null && !relations.isEmpty()) { @@ -73,6 +79,9 @@ public class EntityExportData> { if (attributes != null && !attributes.isEmpty()) { attributes.values().forEach(list -> list.sort(attrComparator)); } + if (calculatedFields != null && !calculatedFields.isEmpty()) { + calculatedFields.sort(calculatedFieldsComparator); + } return this; } @@ -96,4 +105,9 @@ public class EntityExportData> { return relations != null; } + @JsonIgnore + public boolean hasCalculatedFields() { + return calculatedFields != null; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 46dc6860ee..078ca29e28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -25,7 +25,10 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityExportSettings { + private boolean exportRelations; private boolean exportAttributes; private boolean exportCredentials; + private boolean exportCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 2abdb55b65..d49801537c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -25,8 +25,11 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityImportSettings { + private boolean findExistingByName; private boolean updateRelations; private boolean saveAttributes; private boolean saveCredentials; + private boolean saveCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 5a02b18364..6d1ac2283b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -17,13 +17,18 @@ package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; +import java.io.Serial; import java.io.Serializable; @Data public class VersionCreateConfig implements Serializable { + + @Serial private static final long serialVersionUID = 1223723167716612772L; private boolean saveRelations; private boolean saveAttributes; private boolean saveCredentials; + private boolean saveCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 0ae263dada..7f3ce89372 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -23,5 +23,6 @@ public class VersionLoadConfig { private boolean loadRelations; private boolean loadAttributes; private boolean loadCredentials; + private boolean loadCalculatedFields; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 9be8a126b2..06a0489037 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.tenant.profile; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.Min; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -135,11 +136,17 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; - private long maxCalculatedFieldsPerEntity; - private long maxArgumentsPerCF; - private long maxDataPointsPerRollingArg; - private long maxStateSizeInKBytes; - private long maxSingleValueArgumentSizeInKBytes; + @Schema(example = "5") + private long maxCalculatedFieldsPerEntity = 5; + @Schema(example = "10") + private long maxArgumentsPerCF = 10; + @Min(value = 1, message = "must be at least 1") + @Schema(example = "1000") + private long maxDataPointsPerRollingArg = 1000; + @Schema(example = "32") + private long maxStateSizeInKBytes = 32; + @Schema(example = "2") + private long maxSingleValueArgumentSizeInKBytes = 2; @Override public long getProfileThreshold(ApiUsageRecordKey key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java index b1f6c27fd8..44ca79cabb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.tenant.profile; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; import lombok.Data; import java.io.Serializable; @@ -27,6 +28,7 @@ public class TenantProfileData implements Serializable { private static final long serialVersionUID = -3642550257035920976L; + @Valid @Schema(description = "Complex JSON object that contains profile settings: max devices, max assets, rate limits, etc.") private TenantProfileConfiguration configuration; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 82ca82e9cc..71c5256203 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.util; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -75,6 +76,13 @@ public class CollectionsUtil { return isEmpty(collection) || collection.contains(element); } + public static HashSet concat(Set set1, Set set2) { + HashSet result = new HashSet<>(); + result.addAll(set1); + result.addAll(set2); + return result; + } + public static boolean isOneOf(V value, V... others) { if (value == null) { return false; diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml new file mode 100644 index 0000000000..f7106e56fd --- /dev/null +++ b/common/edqs/pom.xml @@ -0,0 +1,97 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + common + + org.thingsboard.common + edqs + jar + + ThingsBoard EDQS API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.rocksdb + rocksdbjni + + + org.thingsboard.common + proto + + + org.thingsboard.common + data + + + org.thingsboard.common + util + + + org.thingsboard.common + message + + + org.thingsboard.common + stats + + + org.thingsboard.common + cluster-api + + + org.thingsboard.common + queue + + + org.springframework.boot + spring-boot-starter-web + + + com.github.ben-manes.caffeine + caffeine + + + org.springframework + spring-context-support + + + org.springframework.boot + spring-boot-autoconfigure + + + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + + diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java new file mode 100644 index 0000000000..f7cd51fc38 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class ApiUsageStateData extends BaseEntityData { + + public ApiUsageStateData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.API_USAGE_STATE; + } + + @Override + public String getEntityName() { + return getEntityOwnerName(); + } + + @Override + public String getEntityOwnerName() { + return repo.getOwnerName(fields.getEntityId()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java new file mode 100644 index 0000000000..52e6151ff3 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class AssetData extends ProfileAwareData { + + public AssetData(UUID id) { + super(id); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java new file mode 100644 index 0000000000..10ee17fc75 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java @@ -0,0 +1,180 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString +public abstract class BaseEntityData implements EntityData { + + @Getter + private final UUID id; + @Getter + protected final Map serverAttrMap; + @Getter + private final Map tMap; + + @Getter + @Setter + private volatile UUID customerId; + + @Setter + protected TenantRepo repo; + + @Getter + @Setter + protected volatile T fields; + + public BaseEntityData(UUID id) { + this.id = id; + this.serverAttrMap = new ConcurrentHashMap<>(); + this.tMap = new ConcurrentHashMap<>(); + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE, SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + default -> null; + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return serverAttrMap.put(keyId, value) == null; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return serverAttrMap.remove(keyId) != null; + } + + @Override + public DataPoint getTs(Integer keyId) { + return tMap.get(keyId); + } + + @Override + public boolean putTs(Integer keyId, DataPoint value) { + return tMap.put(keyId, value) == null; + } + + @Override + public boolean removeTs(Integer keyId) { + return tMap.remove(keyId) != null; + } + + @Override + public EntityType getOwnerType() { + return customerId != null ? EntityType.CUSTOMER : EntityType.TENANT; + } + + @Override + public DataPoint getDataPoint(DataKey key, QueryContext ctx) { + return switch (key.type()) { + case TIME_SERIES -> getTs(key.keyId()); + case ATTRIBUTE, SERVER_ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE -> getAttr(key.keyId(), key.type()); + case ENTITY_FIELD -> getField(key, ctx); + default -> throw new RuntimeException(key.type() + " not supported"); + }; + } + + private DataPoint getField(DataKey newKey, QueryContext ctx) { + if (fields == null) { + return null; + } + String key = newKey.key(); + return switch (key) { + case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime()); + case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate()); + case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx)); + default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false); + }; + } + + @Override + public String getField(String name) { + if (fields == null) { + return null; + } + return switch (name) { + case "name" -> getEntityName(); + case "ownerName" -> getEntityOwnerName(); + case "ownerType" -> customerId != null ? EntityType.CUSTOMER.name() : EntityType.TENANT.name(); + case "entityType" -> Optional.ofNullable(getEntityType()).map(EntityType::name).orElse(""); + default -> fields.getAsString(name); + }; + } + + public String getEntityOwnerName() { + return repo.getOwnerName(getCustomerId() == null || CustomerId.NULL_UUID.equals(getCustomerId()) ? null : + new CustomerId(getCustomerId())); + } + + public String getEntityName() { + return getFields().getName(); + } + + private String getRelatedParentId(QueryContext ctx) { + return Optional.ofNullable(ctx.getRelatedParentIdMap().get(getId())) + .map(UUID::toString) + .orElse(""); + } + + @Override + public EntityType getEntityType() { + return null; + } + + @Override + public boolean isEmpty() { + return fields == null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BaseEntityData that = (BaseEntityData) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java new file mode 100644 index 0000000000..bf2a3f6da7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; + +import java.util.Collection; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class CustomerData extends BaseEntityData { + + private final ConcurrentMap>> entitiesById = new ConcurrentHashMap<>(); + + public CustomerData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + + public Collection> getEntities(EntityType entityType) { + var map = entitiesById.get(entityType); + if (map == null) { + return Collections.emptyList(); + } else { + return map.values(); + } + } + + public void addOrUpdate(EntityData ed) { + entitiesById.computeIfAbsent(ed.getEntityType(), et -> new ConcurrentHashMap<>()).put(ed.getId(), ed); + } + + public boolean remove(EntityData ed) { + var map = entitiesById.get(ed.getEntityType()); + if (map != null) { + return map.remove(ed.getId()) != null; + } else { + return false; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java new file mode 100644 index 0000000000..3a3e5c5792 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.edqs.DataPoint; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString(callSuper = true) +public class DeviceData extends ProfileAwareData { + + private final Map clientAttrMap; + private final Map sharedAttrMap; + + public DeviceData(UUID entityId) { + super(entityId); + this.clientAttrMap = new ConcurrentHashMap<>(); + this.sharedAttrMap = new ConcurrentHashMap<>(); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE -> getAttributeDataPoint(keyId); + case SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + case CLIENT_ATTRIBUTE -> clientAttrMap.get(keyId); + case SHARED_ATTRIBUTE -> sharedAttrMap.get(keyId); + default -> throw new RuntimeException(entityKeyType + " not implemented"); + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.put(keyId, value) == null; + case CLIENT_SCOPE -> clientAttrMap.put(keyId, value) == null; + case SHARED_SCOPE -> sharedAttrMap.put(keyId, value) == null; + }; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.remove(keyId) != null; + case CLIENT_SCOPE -> clientAttrMap.remove(keyId) != null; + case SHARED_SCOPE -> sharedAttrMap.remove(keyId) != null; + }; + } + + private DataPoint getAttributeDataPoint(Integer keyId) { + DataPoint dp = serverAttrMap.get(keyId); + if (dp == null) { + dp = sharedAttrMap.get(keyId); + if (dp == null) { + dp = clientAttrMap.get(keyId); + } + } + return dp; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java new file mode 100644 index 0000000000..53ee73f638 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; + +public interface EntityData { + + UUID getId(); + + EntityType getEntityType(); + + UUID getCustomerId(); + + void setCustomerId(UUID customerId); + + void setRepo(TenantRepo repo); + + T getFields(); + + void setFields(T fields); + + DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType); + + boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value); + + boolean removeAttr(Integer keyId, AttributeScope scope); + + DataPoint getTs(Integer keyId); + + boolean putTs(Integer keyId, DataPoint value); + + boolean removeTs(Integer keyId); + + EntityType getOwnerType(); + + DataPoint getDataPoint(DataKey key, QueryContext queryContext); + + String getField(String name); + + boolean isEmpty(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java new file mode 100644 index 0000000000..a13c70557b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class EntityProfileData extends BaseEntityData { + + private final EntityType entityType; + + public EntityProfileData(UUID entityId, EntityType entityType) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java new file mode 100644 index 0000000000..344a2b7049 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class GenericData extends BaseEntityData { + + private final EntityType entityType; + + public GenericData(EntityType entityType, UUID entityId) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java new file mode 100644 index 0000000000..bab0f962c4 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.edqs.fields.ProfileAwareFields; + +import java.util.UUID; + +public abstract class ProfileAwareData extends BaseEntityData { + + public ProfileAwareData(UUID id) { + super(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java new file mode 100644 index 0000000000..e006def91b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; + +import java.util.UUID; + +public record RelationData(UUID fromId, EntityType fromType, UUID toId, EntityType toType, String type, + RelationTypeGroup typeGroup) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java new file mode 100644 index 0000000000..5eb5d7aee4 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.Data; + +@Data +public class RelationInfo { + + private final String type; + private final EntityData target; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java new file mode 100644 index 0000000000..c094e37261 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@NoArgsConstructor +public class RelationsRepo { + + private final ConcurrentMap> fromRelations = new ConcurrentHashMap<>(); + private final ConcurrentMap> toRelations = new ConcurrentHashMap<>(); + + public boolean add(EntityData from, EntityData to, String type) { + boolean addedFromRelation = fromRelations.computeIfAbsent(from.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, to)); + boolean addedToRelation = toRelations.computeIfAbsent(to.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, from)); + return addedFromRelation || addedToRelation; + } + + public Set getFrom(UUID entityId) { + var result = fromRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public Set getTo(UUID entityId) { + var result = toRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public boolean remove(UUID from, UUID to, String type) { + boolean removedFromRelation = false; + boolean removedToRelation = false; + Set fromRelations = this.fromRelations.get(from); + if (fromRelations != null) { + removedFromRelation = fromRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(to) && relationInfo.getType().equals(type)); + } + Set toRelations = this.toRelations.get(to); + if (toRelations != null) { + removedToRelation = toRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(from) && relationInfo.getType().equals(type)); + } + return removedFromRelation || removedToRelation; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java new file mode 100644 index 0000000000..6822856edf --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; + +import java.util.UUID; + +public class TenantData extends BaseEntityData { + + public TenantData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java new file mode 100644 index 0000000000..fd2d099281 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.edqs.DataPoint; + +@RequiredArgsConstructor +public abstract class AbstractDataPoint implements DataPoint { + + @Getter + private final long ts; + + @Override + public String getStr() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public long getLong() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public double getDouble() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public boolean getBool() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public String getJson() { + throw new RuntimeException(NOT_SUPPORTED); + } + + public String toString() { + return valueToString(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java new file mode 100644 index 0000000000..83d91d8f75 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class BoolDataPoint extends AbstractDataPoint { + + @Getter + private final boolean value; + + public BoolDataPoint(long ts, boolean value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.BOOLEAN; + } + + @Override + public boolean getBool() { + return value; + } + + @Override + public String valueToString() { + return Boolean.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java new file mode 100644 index 0000000000..bce9d86875 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import org.thingsboard.server.common.data.kv.DataType; + +public class CompressedJsonDataPoint extends CompressedStringDataPoint { + + public CompressedJsonDataPoint(long ts, byte[] compressedValue) { + super(ts, compressedValue); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java new file mode 100644 index 0000000000..634b63e012 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.util.TbBytePool; +import org.xerial.snappy.Snappy; + +public class CompressedStringDataPoint extends AbstractDataPoint { + + public static final int MIN_STR_SIZE_TO_COMPRESS = 512; + @Getter + private final byte[] compressedValue; + + @SneakyThrows + public CompressedStringDataPoint(long ts, byte[] compressedValue) { + super(ts); + this.compressedValue = TbBytePool.intern(compressedValue); + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @SneakyThrows + @Override + public String getStr() { + return Snappy.uncompressString(compressedValue); + } + + @Override + public String valueToString() { + return getStr(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java new file mode 100644 index 0000000000..21b355bc46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class DoubleDataPoint extends AbstractDataPoint { + + @Getter + private final double value; + + public DoubleDataPoint(long ts, double value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.DOUBLE; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Double.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java new file mode 100644 index 0000000000..3a8d570f43 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.util.TbStringPool; + +public class JsonDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public JsonDataPoint(long ts, String value) { + super(ts); + this.value = TbStringPool.intern(value); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + + @Override + public String getJson() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java new file mode 100644 index 0000000000..7fbe90e814 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class LongDataPoint extends AbstractDataPoint { + + @Getter + private final long value; + + public LongDataPoint(long ts, long value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.LONG; + } + + @Override + public long getLong() { + return value; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Long.toString(value); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java new file mode 100644 index 0000000000..54156500fe --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.util.TbStringPool; + +public class StringDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public StringDataPoint(long ts, String value) { + this(ts, value, true); + } + + public StringDataPoint(long ts, String value, boolean deduplicate) { + super(ts); + this.value = deduplicate ? TbStringPool.intern(value) : value; + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @Override + public String getStr() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java new file mode 100644 index 0000000000..c5696c08f7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -0,0 +1,298 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ExceptionUtil; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.repo.EdqsRepository; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.edqs.state.EdqsStateService; +import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.edqs.EdqsComponent; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + +@EdqsComponent +@Service +@RequiredArgsConstructor +@Slf4j +public class EdqsProcessor implements TbQueueHandler, TbProtoQueueMsg> { + + private final EdqsQueueFactory queueFactory; + private final EdqsConverter converter; + private final EdqsRepository repository; + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + private final ConfigurableApplicationContext applicationContext; + private final EdqsStateService stateService; + + private PartitionedQueueConsumerManager> eventConsumer; + private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; + + private ExecutorService consumersExecutor; + private ExecutorService taskExecutor; + private ScheduledExecutorService scheduler; + private ListeningExecutorService requestExecutor; + + private final VersionsStore versionsStore = new VersionsStore(); + + private final AtomicInteger counter = new AtomicInteger(); + + @Getter + private Consumer errorHandler; + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); + taskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); + requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); + errorHandler = error -> { + if (error instanceof OutOfMemoryError) { + log.error("OOM detected, shutting down"); + repository.clear(); + Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-shutdown")) + .execute(applicationContext::close); + } + }; + + eventConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) + .topic(EdqsQueue.EVENTS.getTopic()) + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + if (consumer.isStopped()) { + return; + } + try { + ToEdqsMsg msg = queueMsg.getValue(); + process(msg, EdqsQueue.EVENTS); + } catch (Exception t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS)) + .consumerExecutor(consumersExecutor) + .taskExecutor(taskExecutor) + .scheduler(scheduler) + .uncaughtErrorHandler(errorHandler) + .build(); + stateService.init(eventConsumer); + + responseTemplate = queueFactory.createEdqsResponseTemplate(); + } + + @AfterStartUp(order = 1) + public void start() { + responseTemplate.launch(this); + } + + @EventListener + public void onPartitionsChange(PartitionChangeEvent event) { + if (event.getServiceType() != ServiceType.EDQS) { + return; + } + try { + Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); + Set partitions = newPartitions.stream() + .map(tpi -> tpi.withUseInternalPartition(true)) + .collect(Collectors.toSet()); + + stateService.process(withTopic(partitions, EdqsQueue.STATE.getTopic())); + // eventsConsumer's partitions are updated by stateService + responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); // FIXME: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template + + Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); + if (CollectionsUtil.isNotEmpty(oldPartitions)) { + Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() + .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); + if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + } + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId); + return partition != null && removedPartitions.contains(partition); + }); + } + } catch (Throwable t) { + log.error("Failed to handle partition change event {}", event, t); + } + } + + @Override + public ListenableFuture> handle(TbProtoQueueMsg queueMsg) { + ToEdqsMsg toEdqsMsg = queueMsg.getValue(); + return requestExecutor.submit(() -> { + EdqsRequest request; + TenantId tenantId; + CustomerId customerId; + try { + request = Objects.requireNonNull(JacksonUtil.fromString(toEdqsMsg.getRequestMsg().getValue(), EdqsRequest.class)); + tenantId = getTenantId(toEdqsMsg); + customerId = getCustomerId(toEdqsMsg); + } catch (Exception e) { + log.error("Failed to parse request msg: {}", toEdqsMsg, e); + throw e; + } + + EdqsResponse response = processRequest(tenantId, customerId, request); + return new TbProtoQueueMsg<>(queueMsg.getKey(), FromEdqsMsg.newBuilder() + .setResponseMsg(TransportProtos.EdqsResponseMsg.newBuilder() + .setValue(JacksonUtil.toString(response)) + .build()) + .build(), queueMsg.getHeaders()); + }); + } + + private EdqsResponse processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + EdqsResponse response = new EdqsResponse(); + try { + if (request.getEntityDataQuery() != null) { + PageData result = repository.findEntityDataByQuery(tenantId, customerId, + request.getEntityDataQuery(), false); + response.setEntityDataQueryResult(result.mapData(QueryResult::toOldEntityData)); + } else if (request.getEntityCountQuery() != null) { + long result = repository.countEntitiesByQuery(tenantId, customerId, request.getEntityCountQuery(), tenantId.isSysTenantId()); + response.setEntityCountQueryResult(result); + } + log.trace("[{}] Request: {}, response: {}", tenantId, request, response); + } catch (Throwable e) { + log.error("[{}] Failed to process request: {}", tenantId, request, e); + response.setError(ExceptionUtil.getMessage(e)); + } + return response; + } + + public void process(ToEdqsMsg edqsMsg, EdqsQueue queue) { + log.trace("Processing message: {}", edqsMsg); + if (edqsMsg.hasEventMsg()) { + EdqsEventMsg eventMsg = edqsMsg.getEventMsg(); + TenantId tenantId = getTenantId(edqsMsg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + String key = eventMsg.getKey(); + Long version = eventMsg.hasVersion() ? eventMsg.getVersion() : null; + + if (version != null) { + if (!versionsStore.isNew(key, version)) { + return; + } + } else if (!ObjectType.unversionedTypes.contains(objectType)) { + log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key); + } + if (queue != EdqsQueue.STATE) { + stateService.save(tenantId, objectType, key, eventType, edqsMsg); + } + + EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray()); + log.debug("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + int count = counter.incrementAndGet(); + if (count % 100000 == 0) { + log.info("Processed {} events", count); + } + + EdqsEvent event = EdqsEvent.builder() + .tenantId(tenantId) + .objectType(objectType) + .eventType(eventType) + .object(object) + .build(); + repository.processEvent(event); + } + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + private CustomerId getCustomerId(ToEdqsMsg edqsMsg) { + if (edqsMsg.getCustomerIdMSB() != 0 && edqsMsg.getCustomerIdLSB() != 0) { + return new CustomerId(new UUID(edqsMsg.getCustomerIdMSB(), edqsMsg.getCustomerIdLSB())); + } else { + return null; + } + } + + @PreDestroy + public void destroy() throws InterruptedException { + eventConsumer.stop(); + eventConsumer.awaitStop(); + responseTemplate.stop(); + stateService.stop(); + + consumersExecutor.shutdownNow(); + taskExecutor.shutdownNow(); + scheduler.shutdownNow(); + requestExecutor.shutdownNow(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java new file mode 100644 index 0000000000..be1f0481be --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; + +@Slf4j +public class EdqsProducer { + + private final EdqsQueue queue; + private final EdqsPartitionService partitionService; + private final TopicService topicService; + + private final TbQueueProducer> producer; + + @Builder + public EdqsProducer(EdqsQueue queue, + EdqsPartitionService partitionService, + TopicService topicService, + TbQueueProducer> producer) { + this.queue = queue; + this.partitionService = partitionService; + this.topicService = topicService; + this.producer = producer; + } + + public void send(TenantId tenantId, ObjectType type, String key, ToEdqsMsg msg) { + String topic = topicService.buildTopicName(queue.getTopic()); + TbQueueCallback callback = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); + } + + @Override + public void onFailure(Throwable t) { + if (t instanceof RecordTooLargeException) { + if (!log.isDebugEnabled()) { + log.warn("[{}][{}][{}] Failed to publish msg to {}", tenantId, type, key, topic, t); // not logging the whole message + return; + } + } + log.warn("[{}][{}][{}] Failed to publish msg to {}: {}", tenantId, type, key, topic, msg, t); + } + }; + if (producer instanceof TbKafkaProducerTemplate> kafkaProducer) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .partition(partitionService.resolvePartition(tenantId)) + .useInternalPartition(true) + .build(); + kafkaProducer.send(tpi, key, new TbProtoQueueMsg<>(null, msg), callback); // specifying custom key for compaction + } else { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .build(); + producer.send(tpi, new TbProtoQueueMsg<>(null, msg), callback); + } + } + + public void stop() { + producer.stop(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java new file mode 100644 index 0000000000..6d9a672e2c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyType; + +public record DataKey(EntityKeyType type, String key, Integer keyId) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java new file mode 100644 index 0000000000..9c73c3ed9e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +public class EdqsCountQuery extends EdqsQuery { + + @Builder + EdqsCountQuery(EntityFilter entityFilter, boolean hasKeyFilters, List keyFilters) { + super(entityFilter, hasKeyFilters, keyFilters); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java new file mode 100644 index 0000000000..8e118c6e58 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.util.CollectionsUtil; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Getter +public class EdqsDataQuery extends EdqsQuery { + + private final int pageSize; + private final int page; + private final boolean hasTextSearch; + private final String textSearch; + private final boolean defaultSort; + private final DataKey sortKey; + private final EntityDataSortOrder.Direction sortDirection; + private final List entityFields; + private final List latestValues; + + @Builder + public EdqsDataQuery(EntityFilter entityFilter, List keyFilters, + int pageSize, int page, String textSearch, DataKey sortKey, EntityDataSortOrder.Direction sortDirection, + List entityFields, List latestValues) { + super(entityFilter, CollectionsUtil.isNotEmpty(keyFilters), keyFilters); + this.pageSize = pageSize; + this.page = page; + this.hasTextSearch = StringUtils.isNotBlank(textSearch); + this.textSearch = textSearch; + this.defaultSort = EntityKeyType.ENTITY_FIELD.equals(sortKey.type()) && "createdTime".equals(sortKey.key()) && EntityDataSortOrder.Direction.DESC.equals(sortDirection); + this.sortKey = sortKey; + this.sortDirection = sortDirection; + this.entityFields = entityFields; + this.latestValues = latestValues; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java new file mode 100644 index 0000000000..67018ebbb8 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; + +public record EdqsFilter(DataKey key, EntityKeyValueType valueType, KeyFilterPredicate predicate) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java new file mode 100644 index 0000000000..aa20c7c306 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +@Data +public abstract class EdqsQuery { + + private final EntityFilter entityFilter; + private final boolean hasKeyFilters; + private final List keyFilters; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java new file mode 100644 index 0000000000..026c470ce6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.edqs.data.EntityData; + +import java.util.UUID; + +@Data +public class SortableEntityData { + + private final EntityData entityData; + private String sortValue; + + public UUID getId(){ + return entityData.getId(); + } + + public EntityId getEntityId() { + return EntityIdFactory.getByTypeAndUuid(entityData.getEntityType(), entityData.getId()); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java new file mode 100644 index 0000000000..b78e49879e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileNames; + private final Pattern pattern; + + public AbstractEntityProfileNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + entityProfileNames = new HashSet<>(getProfileNames(this.filter)); + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && entityProfileNames.contains(ed.getFields().getType()) + && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java new file mode 100644 index 0000000000..301ead7c63 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + private final Pattern pattern; + + public AbstractEntityProfileQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + var profileNamesSet = new HashSet<>(getProfileNames(this.filter)); + for (EntityData dp : repo.getEntitySet(getProfileEntityType())) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + protected abstract EntityType getProfileEntityType(); + + @Override + protected boolean matches(EntityData ed) { + ProfileAwareData profileAwareData = (ProfileAwareData) ed; + return super.matches(ed) && entityProfileIds.contains(profileAwareData.getFields().getProfileId()) + && (pattern == null || pattern.matcher(profileAwareData.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java new file mode 100644 index 0000000000..b34cd8a459 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntitySearchQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; + +public abstract class AbstractEntitySearchQueryProcessor extends AbstractRelationQueryProcessor { + + + public AbstractEntitySearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + @Override + public Set getRootEntities() { + return Set.of(filter.getRootEntity().getId()); + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + public abstract EntityType getEntityType(); + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData target = relationInfo.getTarget(); + return (filter.getRelationType() == null || relationInfo.getType().equals(filter.getRelationType())) && + getEntityType().equals(target.getEntityType()) && super.matches(target); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java new file mode 100644 index 0000000000..e4cded3e3e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collection; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.checkFilters; +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public abstract class AbstractQueryProcessor implements EntityQueryProcessor { + + protected final TenantRepo repository; + protected final QueryContext ctx; + protected final EdqsQuery query; + protected final DataKey sortKey; + protected final T filter; + + public AbstractQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + this.repository = repository; + this.ctx = ctx; + this.query = query; + this.sortKey = query instanceof EdqsDataQuery dataQuery ? dataQuery.getSortKey() : null; + this.filter = filter; + } + + protected SortableEntityData toSortData(EntityData ed) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + return sortData; + } + + protected void process(Collection> entities, Consumer> processor) { + for (EntityData ed : entities) { + if (matches(ed)) { + processor.accept(ed); + } + } + } + + protected static boolean checkCustomerId(UUID customerId, EntityData ed) { + return customerId.equals(ed.getCustomerId()) || (ed.getEntityType() == EntityType.DASHBOARD && + ed.getFields().getAssignedCustomerIds().contains(customerId)); + } + + protected boolean matches(EntityData ed) { + return checkFilters(query, ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java new file mode 100644 index 0000000000..8ee7338a4f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -0,0 +1,170 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; + + +public abstract class AbstractRelationQueryProcessor extends AbstractQueryProcessor { + + public static final int MAXIMUM_QUERY_LEVEL = 100; + + public AbstractRelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + protected abstract Set getRootEntities(); + + protected abstract EntitySearchDirection getDirection(); + + protected abstract int getMaxLevel(); + + protected abstract boolean isFetchLastLevelOnly(); + + protected boolean isMultiRoot() { + return false; + } + + @Override + public List processQuery() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + if (ctx.isTenantUser()) { + return processTenantQuery(entities); + } else { + return processCustomerQuery(entities); + } + } + + @Override + public long count() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + long result = 0; + + if (ctx.isTenantUser()) { + return entities.size(); + } else { + var customerId = ctx.getCustomerId().getId(); + for (EntityData ed : entities) { + if (checkCustomerId(customerId, ed)) { + result++; + } + } + return result; + } + } + + private List processTenantQuery(Set> entities) { + return entities.stream() + .map(this::toSortData) + .toList(); + } + + private List processCustomerQuery(Set> entities) { + var customerId = ctx.getCustomerId().getId(); + List result = new ArrayList<>(); + for (EntityData ed : entities) { + if (checkCustomerId(customerId, ed)) { + result.add(toSortData(ed)); + } + } + return result; + } + + private Set> getEntitiesSet(RelationsRepo relations) { + Set> result = new HashSet<>(); + Set processed = new HashSet<>(); + Queue tasks = new LinkedList<>(); + int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel()); + for (UUID uuid : getRootEntities()) { + tasks.add(new RelationSearchTask(uuid, 0)); + } + while (!tasks.isEmpty()) { + RelationSearchTask task = tasks.poll(); + if (processed.add(task.entityId)) { + var entityLvl = task.lvl + 1; + Set entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId); + if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) { + result.add(task.previous.getTarget()); + } + for (RelationInfo relationInfo : entities) { + var entity = relationInfo.getTarget(); + if (entity.isEmpty()) { + continue; + } + var entityId = entity.getId(); + if (isFetchLastLevelOnly()) { + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl, relationInfo)); + } else if (entityLvl == maxLvl) { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + } + } else { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl)); + } + } + } + } + } + return result; + } + + protected abstract boolean check(RelationInfo relationInfo); + + @RequiredArgsConstructor + private static class RelationSearchTask { + private final UUID entityId; + private final int lvl; + private final RelationInfo previous; + + public RelationSearchTask(UUID entityId, int lvl) { + this(entityId, lvl, null); + } + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java new file mode 100644 index 0000000000..aab1e83879 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; +import java.util.function.Consumer; + +public abstract class AbstractSimpleQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + + public AbstractSimpleQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter); + this.entityType = entityType; + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + var customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + process(customerData.getEntities(entityType), processor); + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(entityType), processor); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java new file mode 100644 index 0000000000..1723ee5f5b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +public abstract class AbstractSingleEntityTypeQueryProcessor extends AbstractQueryProcessor { + + public AbstractSingleEntityTypeQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + super(repository, ctx, query, filter); + } + + @Override + public List processQuery() { + if (ctx.isTenantUser()) { + return processTenantQuery(); + } else { + return processCustomerQuery(ctx.getCustomerId().getId()); + } + } + + @Override + public long count() { + AtomicLong result = new AtomicLong(); + Consumer> counter = ed -> result.incrementAndGet(); + + if (ctx.isIgnorePermissionCheck()) { + processAll(counter); + } else if (ctx.isTenantUser()) { + processAll(counter); + } else { + processCustomerQuery(ctx.getCustomerId().getId(), counter); + } + return result.get(); + } + + protected List processTenantQuery() { + List result = new ArrayList<>(getProbableResultSize()); + processAll(ed -> { + result.add(toSortData(ed)); + }); + return result; + } + + protected List processCustomerQuery(UUID customerId) { + List result = new ArrayList<>(getProbableResultSize()); + processCustomerQuery(customerId, ed -> { + result.add(toSortData(ed)); + }); + return result; + } + + protected abstract void processCustomerQuery(UUID customerId, Consumer> processor); + + protected abstract void processAll(Consumer> processor); + + protected abstract int getProbableResultSize(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java new file mode 100644 index 0000000000..44370292e4 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; +import java.util.function.Consumer; + +public class ApiUsageStateQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + public ApiUsageStateQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (ApiUsageStateFilter) query.getEntityFilter()); + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + process(customerData.getEntities(EntityType.API_USAGE_STATE), processor); + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(EntityType.API_USAGE_STATE), processor); + } + + @Override + protected boolean matches(EntityData ed) { + ApiUsageStateFields entityFields = (ApiUsageStateFields) ed.getFields(); + return super.matches(ed) && (filter.getCustomerId() == null || filter.getCustomerId().equals(entityFields.getEntityId())); + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java new file mode 100644 index 0000000000..2eff8f6d15 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class AssetSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public AssetSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getAssetTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getAssetTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.ASSET_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java new file mode 100644 index 0000000000..cee5edfa9b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class AssetTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public AssetTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetTypeFilter) query.getEntityFilter(), EntityType.ASSET); + } + + @Override + protected String getEntityNameFilter(AssetTypeFilter filter) { + return filter.getAssetNameFilter(); + } + + @Override + protected List getProfileNames(AssetTypeFilter filter) { + return filter.getAssetTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.ASSET_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java new file mode 100644 index 0000000000..3e53c0815f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class DeviceSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public DeviceSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getDeviceTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getDeviceTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.DEVICE_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java new file mode 100644 index 0000000000..44eaf3e74a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class DeviceTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public DeviceTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceTypeFilter) query.getEntityFilter(), EntityType.DEVICE); + } + + @Override + protected String getEntityNameFilter(DeviceTypeFilter filter) { + return filter.getDeviceNameFilter(); + } + + @Override + protected List getProfileNames(DeviceTypeFilter filter) { + return filter.getDeviceTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java new file mode 100644 index 0000000000..965d3e09ca --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EdgeTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EdgeTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeTypeFilter) query.getEntityFilter(), EntityType.EDGE); + } + + @Override + protected String getEntityNameFilter(EdgeTypeFilter filter) { + return filter.getEdgeNameFilter(); + } + + @Override + protected List getProfileNames(EdgeTypeFilter filter) { + return filter.getEdgeTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java new file mode 100644 index 0000000000..e9e174505b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EdgeTypeSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EdgeTypeSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEdgeTypes() == null || filter.getEdgeTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java new file mode 100644 index 0000000000..3a1eebf1e9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final Set entityIds; + + public EntityListQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityListFilter) query.getEntityFilter()); + this.entityType = filter.getEntityType(); + this.entityIds = filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toSet()); + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + processAll(ed -> { + if (checkCustomerId(customerId, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected void processAll(Consumer> processor) { + var map = repository.getEntityMap(entityType); + for (UUID entityId : entityIds) { + EntityData ed = map.get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + } + + @Override + protected int getProbableResultSize() { + return entityIds.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java new file mode 100644 index 0000000000..ec88db4d0f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.regex.Pattern; + +public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Pattern pattern; + + public EntityNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityNameFilter) query.getEntityFilter(), ((EntityNameFilter) query.getEntityFilter()).getEntityType()); + pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityNameFilter()); + } + + @Override + protected boolean matches(EntityData ed) { + return ed.getFields() != null && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java new file mode 100644 index 0000000000..ad3fc0e7b6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.edqs.query.SortableEntityData; + +import java.util.List; + +public interface EntityQueryProcessor { + + List processQuery(); + + long count(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java new file mode 100644 index 0000000000..12fc863566 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityQueryProcessorFactory { + + public static EntityQueryProcessor create(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + return switch (query.getEntityFilter().getType()) { + case SINGLE_ENTITY -> new SingleEntityQueryProcessor(repo, ctx, query); + case ENTITY_LIST -> new EntityListQueryProcessor(repo, ctx, query); + case ENTITY_NAME -> new EntityNameQueryProcessor(repo, ctx, query); + case ENTITY_TYPE -> new EntityTypeQueryProcessor(repo, ctx, query); + case DEVICE_TYPE -> new DeviceTypeQueryProcessor(repo, ctx, query); + case ASSET_TYPE -> new AssetTypeQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_TYPE -> new EntityViewTypeQueryProcessor(repo, ctx, query); + case EDGE_TYPE -> new EdgeTypeQueryProcessor(repo, ctx, query); + case RELATIONS_QUERY -> new RelationQueryProcessor(repo, ctx, query); + case API_USAGE_STATE -> new ApiUsageStateQueryProcessor(repo, ctx, query); + case ASSET_SEARCH_QUERY -> new AssetSearchQueryProcessor(repo, ctx, query); + case DEVICE_SEARCH_QUERY -> new DeviceSearchQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_SEARCH_QUERY -> new EntityViewSearchQueryProcessor(repo, ctx, query); + case EDGE_SEARCH_QUERY -> new EdgeTypeSearchQueryProcessor(repo, ctx, query); + default -> throw new RuntimeException("Not Implemented!"); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java new file mode 100644 index 0000000000..6f6ec3d007 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityTypeQueryProcessor extends AbstractSimpleQueryProcessor { + + public EntityTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityTypeFilter) query.getEntityFilter(), ((EntityTypeFilter) query.getEntityFilter()).getEntityType()); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java new file mode 100644 index 0000000000..80f56c2169 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityViewSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EntityViewSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEntityViewTypes() == null || filter.getEntityViewTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java new file mode 100644 index 0000000000..2ce4e8616b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EntityViewTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EntityViewTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewTypeFilter) query.getEntityFilter(), EntityType.ENTITY_VIEW); + } + + @Override + protected String getEntityNameFilter(EntityViewTypeFilter filter) { + return filter.getEntityViewNameFilter(); + } + + @Override + protected List getProfileNames(EntityViewTypeFilter filter) { + return filter.getEntityViewTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java new file mode 100644 index 0000000000..d4928a5832 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class RelationQueryProcessor extends AbstractRelationQueryProcessor { + + private final boolean hasFilters; + + public RelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (RelationsQueryFilter) query.getEntityFilter()); + this.hasFilters = filter.getFilters() != null && !filter.getFilters().isEmpty(); + } + + @Override + public Set getRootEntities() { + if (filter.isMultiRoot()) { + return filter.getMultiRootEntityIds().stream().map(UUID::fromString).collect(Collectors.toSet()); + } else { + return Set.of(filter.getRootEntity().getId()); + } + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isMultiRoot() { + return filter.isMultiRoot(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + @Override + protected boolean check(RelationInfo relationInfo) { + if (hasFilters) { + for (var f : filter.getFilters()) { + if (((!filter.isNegate() && !f.isNegate()) || (filter.isNegate() && f.isNegate())) == f.getRelationType().equals(relationInfo.getType())) { + if (f.getEntityTypes() == null || f.getEntityTypes().isEmpty() + || f.getEntityTypes().contains(relationInfo.getTarget().getEntityType())) { + return super.matches(relationInfo.getTarget()); + } + } + } + return false; + } else { + return super.matches(relationInfo.getTarget()); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java new file mode 100644 index 0000000000..febf228699 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; +import java.util.function.Consumer; + +public class SingleEntityQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final UUID entityId; + + public SingleEntityQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (SingleEntityFilter) query.getEntityFilter()); + this.entityType = filter.getSingleEntity().getEntityType(); + this.entityId = filter.getSingleEntity().getId(); + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + processAll(ed -> { + if (checkCustomerId(customerId, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected void processAll(Consumer> processor) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java new file mode 100644 index 0000000000..1deaca83a7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Predicate; + +@EdqsComponent +@AllArgsConstructor +@Service +@Slf4j +public class DefaultEdqsRepository implements EdqsRepository { + + private final static ConcurrentMap repos = new ConcurrentHashMap<>(); + private final Optional statsService; + + public TenantRepo get(TenantId tenantId) { + return repos.computeIfAbsent(tenantId, id -> new TenantRepo(id, statsService)); + } + + @Override + public void processEvent(EdqsEvent event) { + if (event.getEventType() == EdqsEventType.DELETED && event.getObjectType() == ObjectType.TENANT) { + log.info("Tenant {} deleted", event.getTenantId()); + repos.remove(event.getTenantId()); + } else { + get(event.getTenantId()).processEvent(event); + } + } + + @Override + public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("countEntitiesByQuery done in {} ms", timingMs); + return result; + } + + @Override + public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, + EntityDataQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + var result = get(tenantId).findEntityDataByQuery(customerId, query, ignorePermissionCheck); + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("findEntityDataByQuery done in {} ms", timingMs); + return result; + } + + @Override + public void clearIf(Predicate predicate) { + repos.keySet().removeIf(predicate); + } + + @Override + public void clear() { + repos.clear(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java new file mode 100644 index 0000000000..3d9f2ab8df --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +import java.util.function.Predicate; + +public interface EdqsRepository { + + void processEvent(EdqsEvent event); + + long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck); + + PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck); + + void clearIf(Predicate predicate); + + void clear(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java new file mode 100644 index 0000000000..0a79b300a9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class KeyDictionary { + + private static final ConcurrentMap keyToIdDict = new ConcurrentHashMap<>(); + private static final ConcurrentMap idToKeyDict = new ConcurrentHashMap<>(); + private static final AtomicInteger keySeq = new AtomicInteger(); + + public static Integer get(String key) { + return keyToIdDict.computeIfAbsent(key, __ -> { + int keyId = keySeq.incrementAndGet(); + idToKeyDict.put(keyId, key); + return keyId; + }); + } + + public static String get(Integer keyId) { + return idToKeyDict.get(keyId); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java new file mode 100644 index 0000000000..0c55bc50dd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -0,0 +1,439 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.ApiUsageStateData; +import org.thingsboard.server.edqs.data.AssetData; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.GenericData; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.data.TenantData; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessor; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessorFactory; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.edqs.util.RepositoryUtils; +import org.thingsboard.server.edqs.util.TbStringPool; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_ASC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_DESC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.resolveEntityType; + +@Slf4j +public class TenantRepo { + + public static final Comparator> CREATED_TIME_COMPARATOR = Comparator.comparingLong(ed -> ed.getFields().getCreatedTime()); + public static final Comparator> CREATED_TIME_AND_ID_COMPARATOR = CREATED_TIME_COMPARATOR + .thenComparing(EntityData::getId); + public static final Comparator> CREATED_TIME_AND_ID_DESC_COMPARATOR = CREATED_TIME_AND_ID_COMPARATOR.reversed(); + + private final ConcurrentMap>> entitySetByType = new ConcurrentHashMap<>(); + private final ConcurrentMap>> entityMapByType = new ConcurrentHashMap<>(); + private final ConcurrentMap relations = new ConcurrentHashMap<>(); + + private final Lock entityUpdateLock = new ReentrantLock(); + + private final TenantId tenantId; + private final Optional edqsStatsService; + + public TenantRepo(TenantId tenantId, Optional edqsStatsService) { + this.tenantId = tenantId; + this.edqsStatsService = edqsStatsService; + } + + public void processEvent(EdqsEvent event) { + EdqsObject edqsObject = event.getObject(); + log.trace("[{}] Processing event: {}", tenantId, event); + if (event.getEventType() == EdqsEventType.UPDATED) { + addOrUpdate(edqsObject); + } else if (event.getEventType() == EdqsEventType.DELETED) { + remove(edqsObject); + } + } + + public void addOrUpdate(EdqsObject object) { + if (object instanceof EntityRelation relation) { + addOrUpdateRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + addOrUpdateAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + addOrUpdateLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + addOrUpdateEntity(entity); + } + } + + public void remove(EdqsObject object) { + if (object instanceof EntityRelation relation) { + removeRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + removeAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + removeLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + removeEntity(entity); + } + } + + private void addOrUpdateRelation(EntityRelation entity) { + entityUpdateLock.lock(); + try { + if (RelationTypeGroup.COMMON.equals(entity.getTypeGroup())) { + RelationsRepo repo = relations.computeIfAbsent(entity.getTypeGroup(), tg -> new RelationsRepo()); + EntityData from = getOrCreate(entity.getFrom()); + EntityData to = getOrCreate(entity.getTo()); + boolean added = repo.add(from, to, TbStringPool.intern(entity.getType())); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); + } + } else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) { + if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) { + ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entity.getFrom().getId(), CustomerData::new)) + .addOrUpdate(getEntityMap(EntityType.DASHBOARD).get(entity.getTo().getId())); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + private void removeRelation(EntityRelation entityRelation) { + if (RelationTypeGroup.COMMON.equals(entityRelation.getTypeGroup())) { + RelationsRepo relationsRepo = relations.get(entityRelation.getTypeGroup()); + if (relationsRepo != null) { + boolean removed = relationsRepo.remove(entityRelation.getFrom().getId(), entityRelation.getTo().getId(), entityRelation.getType()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); + } + } + } else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) { + if (EntityRelation.CONTAINS_TYPE.equals(entityRelation.getType()) && entityRelation.getFrom().getEntityType() == EntityType.CUSTOMER) { + ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entityRelation.getFrom().getId(), CustomerData::new)) + .remove(getEntityMap(EntityType.DASHBOARD).get(entityRelation.getTo().getId())); + } + } + } + + private void addOrUpdateEntity(Entity entity) { + entityUpdateLock.lock(); + try { + log.trace("[{}] addOrUpdateEntity: {}", tenantId, entity); + EntityFields fields = entity.getFields(); + UUID entityId = fields.getId(); + EntityType entityType = entity.getType(); + + EntityData entityData = getOrCreate(entityType, entityId); + processFields(fields); + EntityFields oldFields = entityData.getFields(); + entityData.setFields(fields); + if (oldFields == null) { + getEntitySet(entityType).add(entityData); + } + + UUID newCustomerId = fields.getCustomerId(); + UUID oldCustomerId = entityData.getCustomerId(); + entityData.setCustomerId(newCustomerId); + if (entityIdMismatch(oldCustomerId, newCustomerId)) { + if (oldCustomerId != null) { + CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); + if (old != null) { + old.remove(entityData); + } + } + if (newCustomerId != null) { + CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); + newData.addOrUpdate(entityData); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void removeEntity(Entity entity) { + entityUpdateLock.lock(); + try { + UUID entityId = entity.getFields().getId(); + EntityType entityType = entity.getType(); + EntityData removed = getEntityMap(entityType).remove(entityId); + if (removed != null) { + if (removed.getFields() != null) { + getEntitySet(entityType).remove(removed); + } + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + UUID customerId = removed.getCustomerId(); + if (customerId != null) { + CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + customerData.remove(removed); + } + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void addOrUpdateAttribute(AttributeKv attributeKv) { + var entityData = getOrCreate(attributeKv.getEntityId()); + if (entityData != null) { + Integer keyId = KeyDictionary.get(attributeKv.getKey()); + boolean added = entityData.putAttr(keyId, attributeKv.getScope(), attributeKv.getDataPoint()); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeAttribute(AttributeKv attributeKv) { + var entityData = get(attributeKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeAttr(KeyDictionary.get(attributeKv.getKey()), attributeKv.getScope()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); + } + } + } + + public void addOrUpdateLatestKv(LatestTsKv latestTsKv) { + var entityData = getOrCreate(latestTsKv.getEntityId()); + if (entityData != null) { + Integer keyId = KeyDictionary.get(latestTsKv.getKey()); + boolean added = entityData.putTs(keyId, latestTsKv.getDataPoint()); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeLatestKv(LatestTsKv latestTsKv) { + var entityData = get(latestTsKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeTs(KeyDictionary.get(latestTsKv.getKey())); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); + } + } + } + + public void processFields(EntityFields fields) { + if (fields instanceof AssetFields assetFields) { + assetFields.setType(TbStringPool.intern(assetFields.getType())); + } + } + + public ConcurrentMap> getEntityMap(EntityType entityType) { + return entityMapByType.computeIfAbsent(entityType, et -> new ConcurrentHashMap<>()); + } + + //TODO: automatically remove entities that has nothing except the ID. + private EntityData getOrCreate(EntityId entityId) { + return getOrCreate(entityId.getEntityType(), entityId.getId()); + } + + private EntityData getOrCreate(EntityType entityType, UUID entityId) { + return getEntityMap(entityType).computeIfAbsent(entityId, id -> { + log.debug("[{}] Adding {} {}", tenantId, entityType, id); + EntityData entityData = constructEntityData(entityType, entityId); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); + return entityData; + }); + } + + private EntityData get(EntityId entityId) { + return getEntityMap(entityId.getEntityType()).get(entityId.getId()); + } + + private EntityData constructEntityData(EntityType entityType, UUID id) { + EntityData entityData = switch (entityType) { + case DEVICE -> new DeviceData(id); + case ASSET -> new AssetData(id); + case DEVICE_PROFILE, ASSET_PROFILE -> new EntityProfileData(id, entityType); + case CUSTOMER -> new CustomerData(id); + case TENANT -> new TenantData(id); + case API_USAGE_STATE -> new ApiUsageStateData(id); + default -> new GenericData(entityType, id); + }; + entityData.setRepo(this); + return entityData; + } + + private static boolean entityIdMismatch(UUID oldOrNull, UUID newOrNull) { + if (oldOrNull == null) { + return newOrNull != null; + } else { + return !oldOrNull.equals(newOrNull); + } + } + + public Set> getEntitySet(EntityType entityType) { + return entitySetByType.computeIfAbsent(entityType, et -> new ConcurrentSkipListSet<>(CREATED_TIME_AND_ID_DESC_COMPARATOR)); + } + + public PageData findEntityDataByQuery(CustomerId customerId, EntityDataQuery oldQuery, boolean ignorePermissionCheck) { + EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return sortAndConvert(query, queryProcessor.processQuery(), ctx); + } + + public long countEntitiesByQuery(CustomerId customerId, EntityCountQuery oldQuery, boolean ignorePermissionCheck) { + EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return queryProcessor.count(); + } + + private PageData sortAndConvert(EdqsDataQuery query, List data, QueryContext ctx) { + int totalSize = data.size(); + int totalPages = (int) Math.ceil((float) totalSize / query.getPageSize()); + int offset = query.getPage() * query.getPageSize(); + if (offset > totalSize) { + return new PageData<>(Collections.emptyList(), totalPages, totalSize, false); + } else { + Comparator comparator = EntityDataSortOrder.Direction.ASC.equals(query.getSortDirection()) ? SORT_ASC : SORT_DESC; + long startTs = System.nanoTime(); +// IMPLEMENTATION THAT IS BASED ON PRIORITY_QUEUE +// var requiredSize = Math.min(offset + query.getPageSize(), totalSize); +// PriorityQueue topN = new PriorityQueue<>(requiredSize, comparator.reversed()); +// for (SortableEntityData item : data) { +// topN.add(item); +// if (topN.size() > requiredSize) { +// topN.poll(); +// } +// } +// List result = new ArrayList<>(topN); +// Collections.reverse(result); +// result = result.subList(offset, requiredSize); +// IMPLEMENTATION THAT IS BASED ON TREE SET (For offset + query.getPageSize() << totalSize) + var requiredSize = Math.min(offset + query.getPageSize(), totalSize); + TreeSet topNSet = new TreeSet<>(comparator); + for (SortableEntityData sp : data) { + topNSet.add(sp); + if (topNSet.size() > requiredSize) { + topNSet.pollLast(); + } + } + var result = topNSet.stream().skip(offset).limit(query.getPageSize()).collect(Collectors.toList()); +// IMPLEMENTATION THAT IS BASED ON TIM SORT (For offset + query.getPageSize() > totalSize / 2) +// data.sort(comparator); +// var result = data.subList(offset, endIndex); + log.trace("EDQ Sorted in {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTs)); + return new PageData<>(toQueryResult(result, query, ctx), totalPages, totalSize, totalSize > requiredSize); + } + } + + private List toQueryResult(List data, EdqsDataQuery query, QueryContext ctx) { + long ts = System.currentTimeMillis(); + List results = new ArrayList<>(data.size()); + for (SortableEntityData entityData : data) { + Map> latest = new HashMap<>(); + for (var key : query.getEntityFields()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(EntityKeyType.ENTITY_FIELD, t -> new HashMap<>()).put(key.key(), v); + } + for (var key : query.getLatestValues()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(key.type(), t -> new HashMap<>()).put(KeyDictionary.get(key.keyId()), v); + } + + results.add(new QueryResult(entityData.getEntityId(), latest)); + } + return results; + } + + private QueryContext buildContext(CustomerId customerId, EntityFilter filter, boolean ignorePermissionCheck) { + return new QueryContext(tenantId, customerId, resolveEntityType(filter), ignorePermissionCheck); + } + + public TenantId getTenantId() { + return tenantId; + } + + + public RelationsRepo getRelations(RelationTypeGroup relationTypeGroup) { + return relations.computeIfAbsent(relationTypeGroup, type -> new RelationsRepo()); + } + + public String getOwnerName(EntityId ownerId) { + if (ownerId == null || (EntityType.CUSTOMER.equals(ownerId.getEntityType()) && CustomerId.NULL_UUID.equals(ownerId.getId()))) { + ownerId = tenantId; + } + return getEntityName(ownerId); + } + + private String getEntityName(EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + return switch (entityType) { + case CUSTOMER, TENANT -> getEntityMap(entityType).get(entityId.getId()).getFields().getName(); + default -> throw new RuntimeException("Unsupported entity type: " + entityType); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java new file mode 100644 index 0000000000..94e9437650 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; + +@Service +@RequiredArgsConstructor +public class EdqsPartitionService { + + private final HashPartitionService hashPartitionService; + private final EdqsConfig edqsConfig; + + public Integer resolvePartition(TenantId tenantId) { + if (edqsConfig.getPartitioningStrategy() == EdqsPartitioningStrategy.TENANT) { + return hashPartitionService.resolvePartitionIndex(tenantId.getId(), edqsConfig.getPartitions()); + } else { + return null; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java new file mode 100644 index 0000000000..ee7b058a8a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; + +import java.util.Set; + +public interface EdqsStateService { + + void init(PartitionedQueueConsumerManager> eventConsumer); + + void process(Set partitions); + + void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); + + boolean isReady(); + + void stop(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java new file mode 100644 index 0000000000..80b6eebf5c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueStateService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; + +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@RequiredArgsConstructor +@KafkaEdqsComponent +@Slf4j +public class KafkaEdqsStateService implements EdqsStateService { + + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + private final EdqsQueueFactory queueFactory; + private final TopicService topicService; + @Autowired @Lazy + private EdqsProcessor edqsProcessor; + + private PartitionedQueueConsumerManager> stateConsumer; + private QueueStateService, TbProtoQueueMsg> queueStateService; + private QueueConsumerManager> eventsToBackupConsumer; + private EdqsProducer stateProducer; + + private final VersionsStore versionsStore = new VersionsStore(); + private final AtomicInteger stateReadCount = new AtomicInteger(); + private final AtomicInteger eventsReadCount = new AtomicInteger(); + private Boolean ready; + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + stateConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) + .topic(EdqsQueue.STATE.getTopic()) + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + edqsProcessor.process(msg, EdqsQueue.STATE); + if (stateReadCount.incrementAndGet() % 100000 == 0) { + log.info("[state] Processed {} msgs", stateReadCount.get()); + } + } catch (Exception e) { + log.error("Failed to process message: {}", queueMsg, e); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.STATE)) + .consumerExecutor(eventConsumer.getConsumerExecutor()) + .taskExecutor(eventConsumer.getTaskExecutor()) + .scheduler(eventConsumer.getScheduler()) + .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) + .build(); + queueStateService = new QueueStateService<>(); + queueStateService.init(stateConsumer, eventConsumer); + + eventsToBackupConsumer = QueueConsumerManager.>builder() + .name("edqs-events-to-backup-consumer") + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + if (consumer.isStopped()) { + return; + } + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + + if (msg.hasEventMsg()) { + EdqsEventMsg eventMsg = msg.getEventMsg(); + String key = eventMsg.getKey(); + int count = eventsReadCount.incrementAndGet(); + if (count % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", count); + } + if (eventMsg.hasVersion()) { + if (!versionsStore.isNew(key, eventMsg.getVersion())) { + continue; + } + } + + TenantId tenantId = getTenantId(msg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + log.trace("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + stateProducer.send(tenantId, objectType, key, msg); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group + .consumerExecutor(eventConsumer.getConsumerExecutor()) + .threadPrefix("edqs-events-to-backup") + .build(); + + stateProducer = EdqsProducer.builder() + .queue(EdqsQueue.STATE) + .partitionService(partitionService) + .topicService(topicService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.STATE)) + .build(); + } + + @Override + public void process(Set partitions) { + if (queueStateService.getPartitions() == null) { + eventsToBackupConsumer.subscribe(); + eventsToBackupConsumer.launch(); + } + queueStateService.update(partitions); + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + // do nothing here, backup is done by events consumer + } + + @Override + public boolean isReady() { + if (ready == null) { + Set partitionsInProgress = queueStateService.getPartitionsInProgress(); + if (partitionsInProgress != null && partitionsInProgress.isEmpty()) { + ready = true; // once true - always true, not to change readiness status on each repartitioning + } + } + return ready != null && ready; + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + @Override + public void stop() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + eventsToBackupConsumer.stop(); + stateProducer.stop(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java new file mode 100644 index 0000000000..383115ddf1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.util.EdqsRocksDb; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.util.Set; + +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + +@Service +@RequiredArgsConstructor +@InMemoryEdqsComponent +@Slf4j +public class LocalEdqsStateService implements EdqsStateService { + + private final EdqsRocksDb db; + @Autowired @Lazy + private EdqsProcessor processor; + + private PartitionedQueueConsumerManager> eventConsumer; + private Set partitions; + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + this.eventConsumer = eventConsumer; + } + + @Override + public void process(Set partitions) { + if (this.partitions == null) { + db.forEach((key, value) -> { + try { + ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); + log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg); + processor.process(edqsMsg, EdqsQueue.STATE); + } catch (Exception e) { + log.error("[{}] Failed to restore value", key, e); + } + }); + log.info("Restore completed"); + } + eventConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic())); + this.partitions = partitions; + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + log.trace("Save to RocksDB: {} {} {} {}", tenantId, type, key, msg); + try { + if (eventType == EdqsEventType.DELETED) { + db.delete(key); + } else { + db.put(key, msg.toByteArray()); + } + } catch (Exception e) { + log.error("[{}] Failed to save event {}", key, msg, e); + } + } + + @Override + public boolean isReady() { + return partitions != null; + } + + @Override + public void stop() { + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java new file mode 100644 index 0000000000..a12a12dbe3 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.stats; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@EdqsComponent +@Service +@Slf4j +@RequiredArgsConstructor +@ConditionalOnProperty(name = "queue.edqs.stats.enabled", havingValue = "true", matchIfMissing = true) +public class EdqsStatsService { + + private final ConcurrentHashMap statsMap = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + @Scheduled(initialDelayString = "${queue.edqs.stats.print-interval-ms:300000}", + fixedDelayString = "${queue.edqs.stats.print-interval-ms:300000}") + private void reportStats() { + if (statsMap.isEmpty()) { + return; + } + String values = statsMap.entrySet().stream() + .map(kv -> "TenantId [" + kv.getKey() + "] stats [" + kv.getValue() + "]") + .collect(Collectors.joining(System.lineSeparator())); + log.info("EDQS Stats: {}", values); + } + + public void reportEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { + statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) + .reportEvent(objectType, eventType); + } + + @Getter + @AllArgsConstructor + static class EdqsStats { + + private final TenantId tenantId; + private final ConcurrentHashMap entityCounters = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + private AtomicInteger getOrCreateObjectCounter(ObjectType objectType) { + return entityCounters.computeIfAbsent(objectType, + type -> statsFactory.createGauge(StatsType.EDQS.getName() + "_object_count", new AtomicInteger(), + "tenantId", tenantId.toString(), "objectType", type.name())); + } + + @Override + public String toString() { + return entityCounters.entrySet().stream() + .map(counters -> counters.getKey().name()+ " total = [" + counters.getValue() + "]") + .collect(Collectors.joining(", ")); + } + + public void reportEvent(ObjectType objectType, EdqsEventType eventType) { + AtomicInteger objectCounter = getOrCreateObjectCounter(objectType); + if (eventType == EdqsEventType.UPDATED){ + objectCounter.incrementAndGet(); + } else if (eventType == EdqsEventType.DELETED) { + objectCounter.decrementAndGet(); + } + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java new file mode 100644 index 0000000000..5b4cd7ac4a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -0,0 +1,253 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.google.protobuf.ByteString; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.FieldsUtil; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedStringDataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.JsonDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; +import org.xerial.snappy.Snappy; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +@Service +@Slf4j +public class EdqsConverter { + + private final Map> converters = new HashMap<>(); + private final Converter defaultConverter = new JsonConverter<>(Entity.class); + + { + converters.put(ObjectType.RELATION, new JsonConverter<>(EntityRelation.class)); + converters.put(ObjectType.ATTRIBUTE_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, AttributeKv attributeKv) { + var proto = TransportProtos.AttributeKvProto.newBuilder() + .setEntityIdMSB(attributeKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(attributeKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(attributeKv.getEntityId().getEntityType())) + .setScope(TransportProtos.AttributeScopeProto.forNumber(attributeKv.getScope().ordinal())) + .setKey(attributeKv.getKey()) + .setVersion(attributeKv.getVersion()); + if (attributeKv.getLastUpdateTs() != null && attributeKv.getValue() != null) { + proto.setDataPoint(toDataPointProto(attributeKv.getLastUpdateTs(), attributeKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public AttributeKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.AttributeKvProto proto = TransportProtos.AttributeKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + AttributeScope scope = AttributeScope.values()[proto.getScope().getNumber()]; + DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; + return AttributeKv.builder() + .entityId(entityId) + .scope(scope) + .key(proto.getKey()) + .version(proto.getVersion()) + .dataPoint(dataPoint) + .build(); + } + }); + converters.put(ObjectType.LATEST_TS_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, LatestTsKv latestTsKv) { + var proto = TransportProtos.LatestTsKvProto.newBuilder() + .setEntityIdMSB(latestTsKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(latestTsKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(latestTsKv.getEntityId().getEntityType())) + .setKey(latestTsKv.getKey()) + .setVersion(latestTsKv.getVersion()); + if (latestTsKv.getTs() != null && latestTsKv.getValue() != null) { + proto.setDataPoint(toDataPointProto(latestTsKv.getTs(), latestTsKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public LatestTsKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.LatestTsKvProto proto = TransportProtos.LatestTsKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; + return LatestTsKv.builder() + .entityId(entityId) + .key(proto.getKey()) + .version(proto.getVersion()) + .dataPoint(dataPoint) + .build(); + } + }); + } + + public static DataPointProto toDataPointProto(long ts, KvEntry kvEntry) { + DataPointProto.Builder proto = DataPointProto.newBuilder(); + proto.setTs(ts); + switch (kvEntry.getDataType()) { + case BOOLEAN -> proto.setBoolV(kvEntry.getBooleanValue().get()); + case LONG -> proto.setLongV(kvEntry.getLongValue().get()); + case DOUBLE -> proto.setDoubleV(kvEntry.getDoubleValue().get()); + case STRING -> { + String strValue = kvEntry.getStrValue().get(); + if (strValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + proto.setStringV(strValue); + } else { + proto.setCompressedStringV(ByteString.copyFrom(compress(strValue))); + } + } + case JSON -> { + String jsonValue = kvEntry.getJsonValue().get(); + if (jsonValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + proto.setJsonV(jsonValue); + } else { + proto.setCompressedJsonV(ByteString.copyFrom(compress(jsonValue))); + } + } + } + return proto.build(); + } + + public static DataPoint fromDataPointProto(DataPointProto proto) { + long ts = proto.getTs(); + if (proto.hasBoolV()) { + return new BoolDataPoint(ts, proto.getBoolV()); + } else if (proto.hasLongV()) { + return new LongDataPoint(ts, proto.getLongV()); + } else if (proto.hasDoubleV()) { + return new DoubleDataPoint(ts, proto.getDoubleV()); + } else if (proto.hasStringV()) { + return new StringDataPoint(ts, proto.getStringV()); + } else if (proto.hasCompressedStringV()) { + return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray()); + } else if (proto.hasJsonV()) { + return new JsonDataPoint(ts, proto.getJsonV()); + } else if (proto.hasCompressedJsonV()) { + return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray()); + } else { + throw new IllegalArgumentException("Unsupported data point proto: " + proto); + } + } + + @SneakyThrows + private static byte[] compress(String value) { + byte[] compressed = Snappy.compress(value); + // TODO: limit the size + log.debug("Compressed {} bytes to {} bytes", value.length(), compressed.length); + return compressed; + } + + public static Entity toEntity(EntityType entityType, Object entity) { + Entity edqsEntity = new Entity(); + edqsEntity.setType(entityType); + edqsEntity.setFields(FieldsUtil.toFields(entity)); + return edqsEntity; + } + + public EdqsObject check(ObjectType type, Object object) { + if (object instanceof EdqsObject edqsObject) { + return edqsObject; + } else { + return toEntity(type.toEntityType(), object); + } + } + + @SuppressWarnings("unchecked") + @SneakyThrows + public byte[] serialize(ObjectType type, T value) { + Converter converter = (Converter) converters.get(type); + if (converter != null) { + return converter.serialize(type, value); + } else { + return defaultConverter.serialize(type, (Entity) value); + } + } + + @SneakyThrows + public EdqsObject deserialize(ObjectType type, byte[] bytes) { + Converter converter = converters.get(type); + if (converter != null) { + return converter.deserialize(type, bytes); + } else { + return defaultConverter.deserialize(type, bytes); + } + } + + @RequiredArgsConstructor + private static class JsonConverter implements Converter { + + private static final ObjectMapper mapper = JsonMapper.builder() + .visibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) + .visibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE) + .visibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE) + .build(); + + private final Class type; + + @SneakyThrows + @Override + public byte[] serialize(ObjectType objectType, T value) { + return mapper.writeValueAsBytes(value); + } + + @SneakyThrows + @Override + public T deserialize(ObjectType objectType, byte[] bytes) { + return mapper.readValue(bytes, this.type); + } + + } + + private interface Converter { + + byte[] serialize(ObjectType type, T value) throws Exception; + + T deserialize(ObjectType type, byte[] bytes) throws Exception; + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java new file mode 100644 index 0000000000..4a991432c7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import org.rocksdb.Options; +import org.rocksdb.WriteOptions; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.nio.file.Files; +import java.nio.file.Path; + +@Component +@InMemoryEdqsComponent +public class EdqsRocksDb extends TbRocksDb { + + @Getter + private boolean isNew; + + public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${user.home}/.rocksdb/edqs}") String path) { + super(path, new Options().setCreateIfMissing(true), new WriteOptions()); + } + + @PostConstruct + @Override + public void init() { + isNew = !Files.exists(Path.of(path)); + super.init(); + } + + @PreDestroy + @Override + public void close() { + super.close(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java new file mode 100644 index 0000000000..970f8585dd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -0,0 +1,392 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsCountQuery; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.KeyDictionary; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; +import static org.thingsboard.server.common.data.StringUtils.equalsAny; +import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithoutQuotes; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.AND; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.OR; + +@Slf4j +public class RepositoryUtils { + + public static final Comparator SORT_ASC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) + .thenComparing(sp -> sp.getId().toString()); + + public static final Comparator SORT_DESC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) + .thenComparing(sp -> sp.getId().toString()).reversed(); + + public static EntityType resolveEntityType(EntityFilter entityFilter) { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> ((SingleEntityFilter) entityFilter).getSingleEntity().getEntityType(); + case ENTITY_LIST -> ((EntityListFilter) entityFilter).getEntityType(); + case ENTITY_NAME -> ((EntityNameFilter) entityFilter).getEntityType(); + case ENTITY_TYPE -> ((EntityTypeFilter) entityFilter).getEntityType(); + case ASSET_TYPE, ASSET_SEARCH_QUERY -> EntityType.ASSET; + case DEVICE_TYPE, DEVICE_SEARCH_QUERY -> EntityType.DEVICE; + case ENTITY_VIEW_TYPE, ENTITY_VIEW_SEARCH_QUERY -> EntityType.ENTITY_VIEW; + case EDGE_TYPE, EDGE_SEARCH_QUERY -> EntityType.EDGE; + case RELATIONS_QUERY -> { + RelationsQueryFilter rgf = (RelationsQueryFilter) entityFilter; + yield rgf.isMultiRoot() ? rgf.getMultiRootEntitiesType() : rgf.getRootEntity().getEntityType(); + } + case API_USAGE_STATE -> EntityType.API_USAGE_STATE; + }; + } + + public static boolean customerUserIsTryingToAccessTenantEntity(QueryContext ctx, EntityFilter entityFilter) { + if (ctx.isTenantUser()) { + return false; + } else { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> { + SingleEntityFilter seFilter = (SingleEntityFilter) entityFilter; + yield isSystemOrTenantEntity(seFilter.getSingleEntity().getEntityType()); + } + case ENTITY_LIST -> { + EntityListFilter elFilter = (EntityListFilter) entityFilter; + yield isSystemOrTenantEntity(elFilter.getEntityType()); + } + case ENTITY_NAME -> { + EntityNameFilter enFilter = (EntityNameFilter) entityFilter; + yield isSystemOrTenantEntity(enFilter.getEntityType()); + } + case ENTITY_TYPE -> { + EntityTypeFilter etFilter = (EntityTypeFilter) entityFilter; + yield isSystemOrTenantEntity(etFilter.getEntityType()); + } + default -> false; + }; + } + } + + private static boolean isSystemOrTenantEntity(EntityType entityType) { + return switch (entityType) { + case DEVICE_PROFILE, ASSET_PROFILE, RULE_CHAIN, TENANT, + TENANT_PROFILE, WIDGET_TYPE, WIDGETS_BUNDLE -> true; + default -> false; + }; + } + + public static EdqsDataQuery toNewQuery(EntityDataQuery oldQuery) { + var query = EdqsDataQuery.builder(); + query.page(oldQuery.getPageLink().getPage()); + query.pageSize(oldQuery.getPageLink().getPageSize()); + query.textSearch(oldQuery.getPageLink().getTextSearch()); + var sortOrder = oldQuery.getPageLink().getSortOrder(); + if (sortOrder != null && toNewKey(sortOrder.getKey()) != null) { + query.sortKey(toNewKey(sortOrder.getKey())); + query.sortDirection(sortOrder.getDirection()); + } else { + query.sortKey(new DataKey(EntityKeyType.ENTITY_FIELD, "createdTime", null)); + query.sortDirection(EntityDataSortOrder.Direction.DESC); + } + query.entityFilter(oldQuery.getEntityFilter()); + query.keyFilters(toKeyFilters(oldQuery.getKeyFilters())); + query.entityFields(toNewKeys(oldQuery.getEntityFields())); + query.latestValues(toNewKeys(oldQuery.getLatestValues())); + return query.build(); + } + + public static EdqsCountQuery toNewQuery(EntityCountQuery oldQuery) { + return EdqsCountQuery.builder() + .entityFilter(oldQuery.getEntityFilter()) + .hasKeyFilters(CollectionsUtil.isNotEmpty(oldQuery.getKeyFilters())) + .keyFilters(toKeyFilters(oldQuery.getKeyFilters())) + .build(); + } + + private static List toKeyFilters(List keyFilters) { + if (keyFilters == null || keyFilters.isEmpty()) { + return Collections.emptyList(); + } else { + List result = new ArrayList<>(); + for (KeyFilter entityFilter : keyFilters) { + var newKey = toNewKey(entityFilter.getKey()); + if (newKey != null) { + result.add(new EdqsFilter(newKey, entityFilter.getValueType(), entityFilter.getPredicate())); + } + } + return result; + } + } + + private static DataKey toNewKey(EntityKey entityKey) { + if (EntityKeyType.ENTITY_FIELD.equals(entityKey.getType())) { + return new DataKey(entityKey.getType(), entityKey.getKey(), null); + } + Integer keyId = KeyDictionary.get(entityKey.getKey()); + if (keyId != null) { + return new DataKey(entityKey.getType(), entityKey.getKey(), keyId); + } else { + log.warn("Missing dictionary key for {}", entityKey.getKey()); + return null; + } + } + + private static List toNewKeys(List entityKeys) { + if (entityKeys == null || entityKeys.isEmpty()) { + return Collections.emptyList(); + } else { + var result = new ArrayList(entityKeys.size()); + for (EntityKey entityKey : entityKeys) { + var newKey = toNewKey(entityKey); + if (newKey != null) { + result.add(newKey); + } + } + return result; + } + } + + public static boolean checkKeyFilters(EntityData entity, List keyFilters) { + for (EdqsFilter keyFilter : keyFilters) { + EntityKeyValueType valueType = keyFilter.valueType(); + if (valueType == null) { + valueType = switch (keyFilter.predicate().getType()) { + case STRING -> EntityKeyValueType.STRING; + case NUMERIC -> EntityKeyValueType.NUMERIC; + case BOOLEAN -> EntityKeyValueType.BOOLEAN; + default -> throw new IllegalStateException(); + }; + } + DataKey dataKey = keyFilter.key(); + DataPoint dp = entity.getDataPoint(dataKey, null); + boolean checkResult = switch (valueType) { + case STRING -> { + String str = dp != null ? dp.valueToString() : null; + yield (dataKey.type() == EntityKeyType.ENTITY_FIELD) ? (str == null || checkKeyFilter(str, keyFilter.predicate())) : + (str != null && checkKeyFilter(str, keyFilter.predicate())); + } + case BOOLEAN -> { + Boolean booleanValue = dp != null ? dp.getBool() : null; + yield booleanValue != null && checkKeyFilter(booleanValue, keyFilter.predicate()); + } + case DATE_TIME, NUMERIC -> { + Double doubleValue = dp != null ? dp.getDouble() : null; + yield doubleValue != null && checkKeyFilter(doubleValue, keyFilter.predicate()); + } + }; + if (!checkResult) { + return false; + } + } + return true; + } + + public static boolean checkKeyFilter(String value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.STRING) { + throw new IllegalStateException("Not implemented"); + } + StringFilterPredicate predicate = (StringFilterPredicate) keyFilterPredicate; + String predicateValue = predicate.getValue().getValue(); + if (StringUtils.isEmpty(predicateValue)) { + return true; + } + if (predicate.isIgnoreCase()) { + predicateValue = predicateValue.toLowerCase(); + value = value.toLowerCase(); + } + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case STARTS_WITH -> value.startsWith(predicateValue); + case ENDS_WITH -> value.endsWith(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case CONTAINS -> value.contains(predicateValue); + case NOT_CONTAINS -> !value.contains(predicateValue); + case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + }; + } + + public static boolean checkKeyFilter(Double value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.NUMERIC) { + throw new IllegalStateException("Not implemented"); + } + NumericFilterPredicate predicate = (NumericFilterPredicate) keyFilterPredicate; + Double predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case GREATER -> value.compareTo(predicateValue) > 0; + case LESS -> value.compareTo(predicateValue) < 0; + case GREATER_OR_EQUAL -> value.compareTo(predicateValue) >= 0; + case LESS_OR_EQUAL -> value.compareTo(predicateValue) <= 0; + }; + } + + public static boolean checkKeyFilter(Boolean value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.BOOLEAN) { + throw new IllegalStateException("Not implemented"); + } + BooleanFilterPredicate predicate = (BooleanFilterPredicate) keyFilterPredicate; + Boolean predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + }; + } + + public static boolean checkComplexKeyFilter(T value, ComplexFilterPredicate filterPredicates, + SimpleKeyFilter simpleKeyFilter) { + if (filterPredicates.getOperation() == AND) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (!simpleKeyFilter.check(value, filterPredicate)) { + return false; + } + } + return true; + } else if (filterPredicates.getOperation() == OR) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (simpleKeyFilter.check(value, filterPredicate)) { + return true; + } + } + return false; + } else { + return false; + } + } + + public static Pattern toSqlLikePattern(String nameFilter) { + if (StringUtils.isNotBlank(nameFilter)) { + boolean percentSymbolOnStart = nameFilter.startsWith("%"); + boolean percentSymbolOnEnd = nameFilter.endsWith("%"); + if (percentSymbolOnStart) { + nameFilter = nameFilter.substring(1); + } + if (percentSymbolOnEnd) { + nameFilter = nameFilter.substring(0, nameFilter.length() - 1); + } + if (percentSymbolOnStart || percentSymbolOnEnd) { + return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE); + } + } + return null; + } + + @FunctionalInterface + public interface SimpleKeyFilter { + + boolean check(T value, KeyFilterPredicate predicate); + + } + + public static TsValue toTsValue(long ts, DataPoint dp) { + if (dp != null) { + return new TsValue(dp.getTs() > 0 ? dp.getTs() : ts, dp.valueToString()); + } else { + return new TsValue(ts, ""); + } + } + + public static String getSortValue(EntityData entity, DataKey sortKey) { + if (sortKey == null) { + return null; + } + switch (sortKey.type()) { + case ENTITY_FIELD -> { + return entity.getField(sortKey.key()); + } + case ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE, SERVER_ATTRIBUTE -> { + var dp = entity.getAttr(sortKey.keyId(), sortKey.type()); + return dp != null ? dp.valueToString() : ""; + } + case TIME_SERIES -> { + var dp = entity.getTs(sortKey.keyId()); + return dp != null ? dp.valueToString() : ""; + } + default -> throw new IllegalStateException("toSortKey is not implemented for type: " + sortKey.type()); + } + } + + public static boolean checkFilters(EdqsQuery query, EntityData entity) { + if (entity == null || entity.getFields() == null) { + return false; // Entity was already removed or not arrived yet; + } + if (query.isHasKeyFilters() && !checkKeyFilters(entity, query.getKeyFilters())) { + return false; + } + if (query instanceof EdqsDataQuery dataQuery) { + return !dataQuery.isHasTextSearch() || checkTextSearch(entity, dataQuery); + } + return true; + } + + private static boolean checkTextSearch(EntityData entityData, EdqsDataQuery query) { + return Stream.concat(query.getEntityFields().stream(), query.getLatestValues().stream()) + .anyMatch(key -> { + DataPoint value = entityData.getDataPoint(key, null); + return value != null && containsIgnoreCase(value.valueToString(), query.getTextSearch()); + }); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java new file mode 100644 index 0000000000..3b135be59c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.google.common.hash.Hashing; +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbBytePool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static byte[] intern(byte[] data) { + if (data == null) { + return null; + } + var checksum = Hashing.sha512().hashBytes(data).toString(); + return pool.computeIfAbsent(checksum, c -> data); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java index 5c4fe185e5..23f2fa2c9e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.edqs.util; import lombok.SneakyThrows; import org.rocksdb.Options; @@ -29,18 +29,24 @@ import java.util.function.BiConsumer; public class TbRocksDb { protected final String path; + private final Options dbOptions; private final WriteOptions writeOptions; - protected final RocksDB db; + protected RocksDB db; static { RocksDB.loadLibrary(); } - public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) throws Exception { + public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) { this.path = path; + this.dbOptions = dbOptions; this.writeOptions = writeOptions; + } + + @SneakyThrows + public void init() { Files.createDirectories(Path.of(path).getParent()); - this.db = RocksDB.open(dbOptions, path); + db = RocksDB.open(dbOptions, path); } @SneakyThrows @@ -68,4 +74,4 @@ public class TbRocksDb { } } -} \ No newline at end of file +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java new file mode 100644 index 0000000000..9c9c3b5b13 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbStringPool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static String intern(String data) { + if (data == null) { + return null; + } + return pool.computeIfAbsent(data, str -> str); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java new file mode 100644 index 0000000000..e52b1bbac9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class VersionsStore { + + private final Cache versions = Caffeine.newBuilder() + .expireAfterWrite(1, TimeUnit.HOURS) + .build(); + + public boolean isNew(String key, Long version) { + AtomicBoolean isNew = new AtomicBoolean(false); + versions.asMap().compute(key, (k, prevVersion) -> { + if (prevVersion == null || prevVersion < version) { + isNew.set(true); + return version; + } else { + if (version < prevVersion) { + log.info("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion); + } + return prevVersion; + } + }); + return isNew.get(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index f1c2236080..4e0f583285 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -375,8 +375,7 @@ public final class TbMsg implements Serializable { protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; - TbMsgBuilder() { - } + TbMsgBuilder() {} TbMsgBuilder(TbMsg tbMsg) { this.queueName = tbMsg.queueName; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java new file mode 100644 index 0000000000..05864fe863 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.edqs; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EdqsApiService { + + ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request); + + boolean isEnabled(); + + void setEnabled(boolean enabled); + + boolean isSupported(); + + boolean isAutoEnable(); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java new file mode 100644 index 0000000000..32ff57e3e0 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.edqs; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EdqsService { + + void onUpdate(TenantId tenantId, EntityId entityId, Object entity); + + void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void onDelete(TenantId tenantId, EntityId entityId); + + void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void processSystemRequest(ToCoreEdqsRequest request); + + void processSystemMsg(ToCoreEdqsMsg request); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 90b82a1e1e..13fd6159fc 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; +import java.io.Serial; import java.util.Optional; /** @@ -33,6 +34,8 @@ import java.util.Optional; */ @Data public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { + + @Serial private static final long serialVersionUID = -5303421482781273062L; private final TenantId tenantId; @@ -66,4 +69,5 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { public MsgType getMsgType() { return MsgType.COMPONENT_LIFE_CYCLE_MSG; } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index 3547ea9120..f31fdfa7a8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -26,7 +26,8 @@ public enum ServiceType { TB_RULE_ENGINE("TB Rule Engine"), TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), - TB_VC_EXECUTOR("TB VC Executor"); + TB_VC_EXECUTOR("TB VC Executor"), + EDQS("TB Entity Data Query Service"); private final String label; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index ddfbd36a33..f09826e9b6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -17,41 +17,48 @@ package org.thingsboard.server.common.msg.queue; import lombok.Builder; import lombok.Getter; -import lombok.ToString; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; -@ToString public class TopicPartitionInfo { private final String topic; private final TenantId tenantId; private final Integer partition; @Getter + private final boolean useInternalPartition; + @Getter private final String fullTopicName; @Getter private final boolean myPartition; @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean useInternalPartition, boolean myPartition) { this.topic = topic; this.tenantId = tenantId; this.partition = partition; + this.useInternalPartition = useInternalPartition; this.myPartition = myPartition; String tmp = topic; if (tenantId != null && !tenantId.isNullUid()) { tmp += ".isolated." + tenantId.getId().toString(); } - if (partition != null) { + if (partition != null && !useInternalPartition) { tmp += "." + partition; } this.fullTopicName = tmp; } + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + this(topic, tenantId, partition, false, myPartition); + } + public TopicPartitionInfo newByTopic(String topic) { - return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); } public String getTopic() { @@ -66,6 +73,18 @@ public class TopicPartitionInfo { return Optional.ofNullable(partition); } + public TopicPartitionInfo withTopic(String topic) { + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); + } + + public static Set withTopic(Set partitions, String topic) { + return partitions.stream().map(tpi -> tpi.withTopic(topic)).collect(Collectors.toSet()); + } + + public TopicPartitionInfo withUseInternalPartition(boolean useInternalPartition) { + return new TopicPartitionInfo(this.topic, this.tenantId, this.partition, useInternalPartition, this.myPartition); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -79,6 +98,16 @@ public class TopicPartitionInfo { @Override public int hashCode() { - return Objects.hash(fullTopicName); + return Objects.hash(fullTopicName, partition); + } + + @Override + public String toString() { + String str = fullTopicName; + if (useInternalPartition) { + str += "[" + partition + "]"; + } + return str; } + } diff --git a/common/pom.xml b/common/pom.xml index c275227018..9a7a836ba5 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -49,6 +49,7 @@ edge-api version-control script + edqs diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index a570dbea89..f1da00d943 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -135,7 +135,7 @@ public class ProtoUtils { builder.setName(msg.getName()); } if (msg.getOldName() != null) { - builder.setName(msg.getOldName()); + builder.setOldName(msg.getOldName()); } return builder.build(); } @@ -171,7 +171,6 @@ public class ProtoUtils { return entityTypeByProtoNumber[entityType.getNumber()]; } - public static TransportProtos.ToEdgeSyncRequestMsgProto toProto(ToEdgeSyncRequest request) { return TransportProtos.ToEdgeSyncRequestMsgProto.newBuilder() .setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 77dab2cb57..38f86bf9db 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -72,6 +72,7 @@ message ServiceInfo { repeated string transports = 6; SystemInfoProto systemInfo = 10; repeated string assignedTenantProfiles = 11; + string label = 12; } message SystemInfoProto { @@ -172,17 +173,47 @@ message AttributeValueProto { optional int64 version = 10; } +message AttributeKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + AttributeScopeProto scope = 4; + string key = 5; + int64 version = 6; + DataPointProto dataPoint = 7; +} + message TsKvProto { int64 ts = 1; KeyValueProto kv = 2; optional int64 version = 3; } +message LatestTsKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + string key = 4; + int64 version = 5; + DataPointProto dataPoint = 6; +} + message TsKvListProto { int64 ts = 1; repeated KeyValueProto kv = 2; } +message DataPointProto { + int64 ts = 1; + optional bool boolV = 2; + optional int64 longV = 3; + optional double doubleV = 4; + optional string stringV = 5; + optional bytes compressedStringV = 6; + optional string jsonV = 7; + optional bytes compressedJsonV = 8; +} + message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -485,6 +516,10 @@ message ImageCacheKeyProto { optional string publicResourceKey = 2; } +message ToEdqsCoreServiceMsg { + bytes value = 1; +} + message LwM2MRegistrationRequestMsg { string tenantId = 1; string endpoint = 2; @@ -774,6 +809,14 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } +message DeviceInactivityTimeoutUpdateProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + int64 inactivityTimeout = 5; +} + message CalculatedFieldTelemetryMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -1586,6 +1629,7 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; + DeviceInactivityTimeoutUpdateProto deviceInactivityTimeoutUpdateMsg = 53; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1604,6 +1648,7 @@ message ToCoreNotificationMsg { ToEdgeSyncRequestMsgProto toEdgeSyncRequest = 11 [deprecated = true]; FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; + ToEdqsCoreServiceMsg toEdqsCoreServiceMsg = 17; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; } @@ -1743,3 +1788,33 @@ message HousekeeperTaskProto { int32 attempt = 50; repeated string errors = 51; } + +message ToEdqsMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 customerIdMSB = 3; + int64 customerIdLSB = 4; + int64 ts = 5; + EdqsEventMsg eventMsg = 6; + EdqsRequestMsg requestMsg = 7; +} + +message FromEdqsMsg { + EdqsResponseMsg responseMsg = 1; +} + +message EdqsEventMsg { + string key = 1; + string objectType = 2; + bytes data = 3; + string eventType = 4; + optional int64 version = 5; +} + +message EdqsRequestMsg { + string value = 1; +} + +message EdqsResponseMsg { + string value = 1; +} diff --git a/common/queue/pom.xml b/common/queue/pom.xml index c0e51318d8..e28ebfbcf0 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -116,6 +116,10 @@ org.apache.curator curator-recipes + + org.xerial.snappy + snappy-java + org.springframework.boot spring-boot-starter-test diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index e4cbc121ba..7e7de64a5c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -31,7 +31,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -95,9 +94,8 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - List topicNames = getFullTopicNames(); - log.info("Subscribing to topics {}", topicNames); - doSubscribe(topicNames); + log.info("Subscribing to {}", partitions); + doSubscribe(partitions); subscribed = true; } records = partitions.isEmpty() ? emptyList() : doPoll(durationInMillis); @@ -169,7 +167,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { - log.info("Unsubscribing and stopping consumer for topics {}", getFullTopicNames()); + log.info("Unsubscribing and stopping consumer for {}", partitions); stopped = true; consumerLock.lock(); try { @@ -190,7 +188,7 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected T decode(R record) throws IOException; - abstract protected void doSubscribe(List topicNames); + abstract protected void doSubscribe(Set partitions); abstract protected void doCommit(); @@ -201,7 +199,9 @@ public abstract class AbstractTbQueueConsumerTemplate i if (partitions == null) { return Collections.emptyList(); } - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + return partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .toList(); } protected boolean isLongPollingSupported() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 6a73155b39..4efb297491 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -211,6 +211,15 @@ public class DefaultTbQueueRequestTemplate send(Request request, long requestTimeoutNs) { + return send(request, requestTimeoutNs, null); + } + + @Override + public ListenableFuture send(Request request, Integer partition) { + return send(request, this.maxRequestTimeoutNs, partition); + } + + private ListenableFuture send(Request request, long requestTimeoutNs, Integer partition) { if (pendingRequests.mappingCount() >= maxPendingRequests) { log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance. Request is {}", maxPendingRequests, request); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); @@ -227,7 +236,7 @@ public class DefaultTbQueueRequestTemplate future, ResponseMetaData responseMetaData) { + void sendToRequestTemplate(Request request, UUID requestId, Integer partition, SettableFuture future, ResponseMetaData responseMetaData) { log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } - requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(requestTemplate.getDefaultTopic()) + .partition(partition) + .useInternalPartition(partition != null) + .build(); + requestTemplate.send(tpi, request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { if (messagesStats != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index 0c925e9334..abd254b269 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -28,6 +28,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueResponseTemplate; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -77,9 +78,18 @@ public class DefaultTbQueueResponseTemplate handler) { - this.responseTemplate.init(); + public void subscribe() { requestTemplate.subscribe(); + } + + @Override + public void subscribe(Set partitions) { + requestTemplate.subscribe(partitions); + } + + @Override + public void launch(TbQueueHandler handler) { + this.responseTemplate.init(); loopExecutor.submit(() -> { while (!stopped) { try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 6a601f4628..14394bbbe9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -43,11 +43,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; import java.util.function.Consumer; -import java.util.stream.Collectors; @Slf4j public class MainQueueConsumerManager { + @Getter protected final QueueKey queueKey; @Getter protected C config; @@ -59,6 +59,7 @@ public class MainQueueConsumerManager uncaughtErrorHandler; private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); private final ReentrantLock lock = new ReentrantLock(); @@ -74,7 +75,8 @@ public class MainQueueConsumerManager> consumerCreator, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, - ExecutorService taskExecutor) { + ExecutorService taskExecutor, + Consumer uncaughtErrorHandler) { this.queueKey = queueKey; this.config = config; this.msgPackProcessor = msgPackProcessor; @@ -82,6 +84,7 @@ public class MainQueueConsumerManager consumerLoop = consumerExecutor.submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); - try { - consumerLoop(consumerTask.getConsumer()); - } catch (Throwable e) { - log.error("Failure in consumer loop", e); - } + consumerLoop(consumerTask.getConsumer()); log.info("[{}] Consumer stopped", consumerTask.getKey()); try { @@ -219,25 +218,33 @@ public class MainQueueConsumerManager consumer) { - while (!stopped && !consumer.isStopped()) { - try { - List msgs = consumer.poll(config.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - processMsgs(msgs, consumer, config); - } catch (Exception e) { - if (!consumer.isStopped()) { - log.warn("Failed to process messages from queue", e); - try { - Thread.sleep(config.getPollInterval()); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + try { + while (!stopped && !consumer.isStopped()) { + try { + List msgs = consumer.poll(config.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + processMsgs(msgs, consumer, config); + } catch (Exception e) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", e); + try { + Thread.sleep(config.getPollInterval()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } - } - if (consumer.isStopped()) { + if (consumer.isStopped()) { + consumer.unsubscribe(); + } + } catch (Throwable t) { + log.error("Failure in consumer loop", t); + if (uncaughtErrorHandler != null) { + uncaughtErrorHandler.accept(t); + } consumer.unsubscribe(); } } @@ -264,10 +271,6 @@ public class MainQueueConsumerManager partitions) { - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]")); - } - public interface MsgPackProcessor { void process(List msgs, TbQueueConsumer consumer, C config) throws Exception; } @@ -291,7 +294,7 @@ public class MainQueueConsumerManager removedPartitions = new HashSet<>(consumers.keySet()); removedPartitions.removeAll(partitions); - log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions)); + log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, addedPartitions, removedPartitions); removePartitions(removedPartitions); addPartitions(addedPartitions, null); } @@ -325,7 +328,7 @@ public class MainQueueConsumerManager partitions) { - log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions)); + log.info("[{}] New partitions: {}", queueKey, partitions); if (partitions.isEmpty()) { if (consumer != null && consumer.isRunning()) { consumer.initiateStop(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java index c94fdd8d3c..1d68cc9c4e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -43,8 +43,8 @@ public class PartitionedQueueConsumerManager extends MainQ public PartitionedQueueConsumerManager(QueueKey queueKey, String topic, long pollInterval, MsgPackProcessor msgPackProcessor, BiFunction> consumerCreator, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, - ExecutorService taskExecutor) { - super(queueKey, QueueConfig.of(true, pollInterval), msgPackProcessor, consumerCreator, consumerExecutor, scheduler, taskExecutor); + ExecutorService taskExecutor, Consumer uncaughtErrorHandler) { + super(queueKey, QueueConfig.of(true, pollInterval), msgPackProcessor, consumerCreator, consumerExecutor, scheduler, taskExecutor, uncaughtErrorHandler); this.topic = topic; this.consumerWrapper = (ConsumerPerPartitionWrapper) super.consumerWrapper; } @@ -52,10 +52,10 @@ public class PartitionedQueueConsumerManager extends MainQ @Override protected void processTask(TbQueueConsumerManagerTask task) { if (task instanceof AddPartitionsTask addPartitionsTask) { - log.info("[{}] Added partitions: {}", queueKey, partitionsToString(addPartitionsTask.partitions())); + log.info("[{}] Added partitions: {}", queueKey, addPartitionsTask.partitions()); consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop()); } else if (task instanceof RemovePartitionsTask removePartitionsTask) { - log.info("[{}] Removed partitions: {}", queueKey, partitionsToString(removePartitionsTask.partitions())); + log.info("[{}] Removed partitions: {}", queueKey, removePartitionsTask.partitions()); consumerWrapper.removePartitions(removePartitionsTask.partitions()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java index 3da1e8fa4b..8870ff2a2c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java @@ -16,16 +16,20 @@ package org.thingsboard.server.queue.common.consumer; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueMsg; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + +@Slf4j public class QueueStateService { private PartitionedQueueConsumerManager stateConsumer; @@ -33,6 +37,9 @@ public class QueueStateService { @Getter private Set partitions; + private final Set partitionsInProgress = ConcurrentHashMap.newKeySet(); + private boolean initialized; + private final ReadWriteLock partitionsLock = new ReentrantReadWriteLock(); public void init(PartitionedQueueConsumerManager stateConsumer, PartitionedQueueConsumerManager eventConsumer) { @@ -63,22 +70,29 @@ public class QueueStateService { } if (!addedPartitions.isEmpty()) { + partitionsInProgress.addAll(addedPartitions); stateConsumer.addPartitions(addedPartitions, partition -> { var readLock = partitionsLock.readLock(); readLock.lock(); try { + partitionsInProgress.remove(partition); + log.info("Finished partition {} (still in progress: {})", partition, partitionsInProgress); + if (partitionsInProgress.isEmpty()) { + log.info("All partitions processed"); + } if (this.partitions.contains(partition)) { - eventConsumer.addPartitions(Set.of(partition.newByTopic(eventConsumer.getTopic()))); + eventConsumer.addPartitions(Set.of(partition.withTopic(eventConsumer.getTopic()))); } } finally { readLock.unlock(); } }); } + initialized = true; } - private Set withTopic(Set partitions, String topic) { - return partitions.stream().map(tpi -> tpi.newByTopic(topic)).collect(Collectors.toSet()); + public Set getPartitionsInProgress() { + return initialized ? partitionsInProgress : null; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java index f51594b48b..28066d9a91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java @@ -76,7 +76,7 @@ public class TbQueueConsumerTask { awaitCompletion(30); } - public void awaitCompletion(long timeoutSec) { + public void awaitCompletion(int timeoutSec) { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 4bcbb5152c..609d3f8eee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.util.AfterContextReady; import java.net.InetAddress; @@ -58,6 +59,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.rule_engine.assigned_tenant_profiles:}") private Set assignedTenantProfiles; + @Autowired + private EdqsConfig edqsConfig; + @Autowired private ApplicationContext applicationContext; @@ -82,6 +86,11 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (!serviceTypes.contains(ServiceType.TB_RULE_ENGINE) || assignedTenantProfiles == null) { assignedTenantProfiles = Collections.emptySet(); } + if (serviceTypes.contains(ServiceType.EDQS)) { + if (StringUtils.isBlank(edqsConfig.getLabel())) { + edqsConfig.setLabel(serviceId); + } + } generateNewServiceInfoWithCurrentSystemInfo(); } @@ -118,6 +127,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); } + builder.setLabel(edqsConfig.getLabel()); return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 27b3835cd0..b42f8cc380 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -20,6 +20,7 @@ import com.google.common.hash.Hashing; import jakarta.annotation.PostConstruct; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -58,7 +59,7 @@ import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; @Slf4j public class HashPartitionService implements PartitionService { - @Value("${queue.core.topic}") + @Value("${queue.core.topic:tb_core}") private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; @@ -76,6 +77,8 @@ public class HashPartitionService implements PartitionService { private String edgeTopic; @Value("${queue.edge.partitions:10}") private Integer edgePartitions; + @Value("${queue.edqs.partitions:12}") + private Integer edqsPartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @@ -135,6 +138,10 @@ public class HashPartitionService implements PartitionService { QueueKey edgeKey = coreKey.withQueueName(EDGE_QUEUE_NAME); partitionSizesMap.put(edgeKey, edgePartitions); partitionTopicsMap.put(edgeKey, edgeTopic); + + QueueKey edqsKey = new QueueKey(ServiceType.EDQS); + partitionSizesMap.put(edqsKey, edqsPartitions); + partitionTopicsMap.put(edqsKey, "edqs"); // placeholder, not used } @AfterStartUp(order = AfterStartUp.QUEUE_INFO_INITIALIZATION) @@ -228,7 +235,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); } } @@ -372,6 +379,11 @@ public class HashPartitionService implements PartitionService { } } + @Override + public boolean isSystemPartitionMine(ServiceType serviceType) { + return isMyPartition(serviceType, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); + } + @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { log.info("Recalculating partitions"); @@ -392,9 +404,9 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { try { - ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); - log.trace("Server responsible for {}[{}] - {}", queueKey, i, serviceInfo != null ? serviceInfo.getServiceId() : "none"); - if (currentService.equals(serviceInfo)) { + List services = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); + log.trace("Server responsible for {}[{}] - {}", queueKey, i, services); + if (services.contains(currentService)) { newPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); } } catch (Exception e) { @@ -408,6 +420,7 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); + Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -428,16 +441,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - changedPartitionsMap.put(queueKey, tpiList); + changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); + oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); } }); if (!changedPartitionsMap.isEmpty()) { changedPartitionsMap.entrySet().stream() .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) - .forEach(this::publishPartitionChangeEvent); + .forEach((serviceType, partitionsMap) -> { + publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); + }); } if (currentOtherServices == null) { @@ -469,13 +482,15 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { - log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { + log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions, oldPartitions); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -483,6 +498,15 @@ public class HashPartitionService implements PartitionService { } } + private Set toTpiList(QueueKey queueKey, List partitions) { + if (partitions == null) { + return Collections.emptySet(); + } + return partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + } + @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); @@ -617,6 +641,8 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(new QueueKey(serviceType).withQueueName(EDGE_QUEUE_NAME), key -> new ArrayList<>()).add(instance); } else if (ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); + } else if (ServiceType.EDQS.equals(serviceType)) { + queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } @@ -625,10 +651,11 @@ public class HashPartitionService implements PartitionService { } } - protected ServiceInfo resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, - Map> responsibleServices) { + @NotNull + protected List resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, + Map> responsibleServices) { if (servers == null || servers.isEmpty()) { - return null; + return Collections.emptyList(); } TenantId tenantId = queueKey.getTenantId(); @@ -656,15 +683,21 @@ public class HashPartitionService implements PartitionService { responsibleServices.put(profileId, responsible); } if (responsible.isEmpty()) { - return null; + return Collections.emptyList(); } servers = responsible; } int hash = hash(tenantId.getId()); - return servers.get(Math.abs((hash + partition) % servers.size())); + ServiceInfo server = servers.get(Math.abs((hash + partition) % servers.size())); + return server != null ? List.of(server) : Collections.emptyList(); + } else if (queueKey.getType() == ServiceType.EDQS) { + List> sets = servers.stream().collect(Collectors.groupingBy(ServiceInfo::getLabel)) + .entrySet().stream().sorted(Map.Entry.comparingByKey()).map(Map.Entry::getValue).toList(); + return sets.get(partition % sets.size()); } else { - return servers.get(partition % servers.size()); + ServiceInfo server = servers.get(partition % servers.size()); + return server != null ? List.of(server) : Collections.emptyList(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b067a9ce92..404b0258c0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -43,6 +43,8 @@ public interface PartitionService { boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); + boolean isSystemPartitionMine(ServiceType serviceType); + List getMyPartitions(QueueKey queueKey); String getTopic(QueueKey queueKey); @@ -65,8 +67,6 @@ public interface PartitionService { Set getOtherServices(ServiceType serviceType); - int resolvePartitionIndex(UUID entityId, int partitions); - void evictTenantInfo(TenantId tenantId); int countTransportsByType(String type); @@ -79,6 +79,8 @@ public interface PartitionService { boolean isManagedByCurrentService(TenantId tenantId); + int resolvePartitionIndex(UUID entityId, int partitions); + int getTotalCalculatedFieldPartitions(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java index 1e6cb01478..ca38959fdd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java @@ -62,6 +62,12 @@ public class QueueKey { this.tenantId = TenantId.SYS_TENANT_ID; } + public QueueKey(ServiceType type, String queueName) { + this.type = type; + this.queueName = queueName; + this.tenantId = TenantId.SYS_TENANT_ID; + } + @Override public String toString() { return "QK(" + queueName + "," + type + "," + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 138e963188..f7a4d2abf6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -19,6 +19,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.ProtocolStringList; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; @@ -68,6 +69,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private Integer zkConnectionTimeout; @Value("${zk.session_timeout_ms}") private Integer zkSessionTimeout; + @Getter @Value("${zk.zk_dir}") private String zkDir; @Value("${zk.recalculate_delay:0}") @@ -80,6 +82,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private final PartitionService partitionService; private ScheduledExecutorService zkExecutorService; + @Getter private CuratorFramework client; private PathChildrenCache cache; private String nodePath; @@ -140,6 +143,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } else { log.info("Received application ready event. Starting current ZK node."); } + subscribeToEvents(); if (client.getState() != CuratorFrameworkState.STARTED) { log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); return; @@ -209,6 +213,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi try { destroyZkClient(); initZkClient(); + subscribeToEvents(); publishCurrentServer(); } catch (Exception e) { log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); @@ -224,7 +229,6 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi client.start(); client.blockUntilConnected(); cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); cache.start(); stopped = false; log.info("ZK client connected"); @@ -236,6 +240,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } + private void subscribeToEvents() { + cache.getListenable().addListener(this); + } + private void unpublishCurrentServer() { try { if (nodePath != null) { @@ -243,25 +251,21 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } catch (Exception e) { log.error("Failed to delete ZK node {}", nodePath, e); - throw new RuntimeException(e); } } private void destroyZkClient() { stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) { - } + unpublishCurrentServer(); CloseableUtils.closeQuietly(cache); CloseableUtils.closeQuietly(client); log.info("ZK client disconnected"); } @PreDestroy - public void destroy() { - destroyZkClient(); + private void destroy() { zkExecutorService.shutdownNow(); + destroyZkClient(); log.info("Stopped discovery service"); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 056fbe44d0..597463300a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -37,12 +38,17 @@ public class PartitionChangeEvent extends TbApplicationEvent { @Getter private final ServiceType serviceType; @Getter - private final Map> partitionsMap; + private final Map> newPartitions; + @Getter + private final Map> oldPartitions; - public PartitionChangeEvent(Object source, ServiceType serviceType, Map> partitionsMap) { + public PartitionChangeEvent(Object source, ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { super(source); this.serviceType = serviceType; - this.partitionsMap = partitionsMap; + this.newPartitions = newPartitions; + this.oldPartitions = oldPartitions; } public Set getCorePartitions() { @@ -53,15 +59,20 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } + public Set getPartitions() { + return newPartitions.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); + } + public Set getCfPartitions() { - return partitionsMap.getOrDefault(QueueKey.CF, Collections.emptySet()); + return newPartitions.getOrDefault(QueueKey.CF, Collections.emptySet()); } private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { - return partitionsMap.entrySet() + return newPartitions.entrySet() .stream() .filter(entry -> serviceType.equals(entry.getKey().getType()) && queueName.equals(entry.getKey().getQueueName())) .flatMap(entry -> entry.getValue().stream()) .collect(Collectors.toSet()); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java new file mode 100644 index 0000000000..c3658d098a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local'))") +public @interface EdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java new file mode 100644 index 0000000000..e4e1e81815 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Data +public class EdqsConfig { + + @Value("${queue.edqs.partitions:12}") + private int partitions; + @Value("${service.edqs.label:}") + private String label; + @Value("#{'${queue.edqs.partitioning_strategy:tenant}'.toUpperCase()}") + private EdqsPartitioningStrategy partitioningStrategy; + + @Value("${queue.edqs.requests_topic:edqs.requests}") + private String requestsTopic; + @Value("${queue.edqs.responses_topic:edqs.responses}") + private String responsesTopic; + @Value("${queue.edqs.poll_interval:125}") + private long pollInterval; + @Value("${queue.edqs.max_pending_requests:10000}") + private int maxPendingRequests; + @Value("${queue.edqs.max_request_timeout:20000}") + private int maxRequestTimeout; + + public String getLabel() { + if (partitioningStrategy == EdqsPartitioningStrategy.NONE) { + label = "all"; // single set for all instances, so that each instance has all partitions + } + return label; + } + + public enum EdqsPartitioningStrategy { + TENANT, NONE + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java new file mode 100644 index 0000000000..d859b50994 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Getter; + +@Getter +public enum EdqsQueue { + + EVENTS("edqs.events", false, false), + STATE("edqs.state", true, true); + + private final String topic; + private final boolean readFromBeginning; + private final boolean stopWhenRead; + + EdqsQueue(String topic, boolean readFromBeginning, boolean stopWhenRead) { + this.topic = topic; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java new file mode 100644 index 0000000000..fed786e120 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface EdqsQueueFactory { + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue); + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group); + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java new file mode 100644 index 0000000000..e414d24fd9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") +public @interface InMemoryEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java new file mode 100644 index 0000000000..0b6cc1909d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.memory.InMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; + +@Component +@InMemoryEdqsComponent +@RequiredArgsConstructor +public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { + + private final InMemoryStorage storage; + private final EdqsConfig edqsConfig; + private final StatsFactory statsFactory; + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueConsumer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return createEdqsMsgConsumer(queue); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + TbQueueConsumer> requestConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueProducer> responseProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getResponsesTopic()); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer) + .responseTemplate(responseProducer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java new file mode 100644 index 0000000000..3a2b282724 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='kafka'))") +public @interface KafkaEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java new file mode 100644 index 0000000000..42ca604841 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; + +import java.util.concurrent.atomic.AtomicInteger; + +@Component +@KafkaEdqsComponent +public class KafkaEdqsQueueFactory implements EdqsQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbKafkaAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; + private final TbKafkaAdmin edqsStateAdmin; + private final EdqsConfig edqsConfig; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbKafkaConsumerStatsService consumerStatsService; + private final TopicService topicService; + private final StatsFactory statsFactory; + + private final AtomicInteger consumerCounter = new AtomicInteger(); + + public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs, + EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider, + TbKafkaConsumerStatsService consumerStatsService, TopicService topicService, + StatsFactory statsFactory) { + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsRequestsConfigs()); + this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs()); + this.kafkaSettings = kafkaSettings; + this.edqsConfig = edqsConfig; + this.serviceInfoProvider = serviceInfoProvider; + this.consumerStatsService = consumerStatsService; + this.topicService = topicService; + this.statsFactory = statsFactory; + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + String consumerGroup = "edqs-" + queue.name().toLowerCase() + "-consumer-group-" + serviceInfoProvider.getServiceId(); + return createEdqsMsgConsumer(queue, consumerGroup); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(queue.getTopic())) + .readFromBeginning(queue.isReadFromBeginning()) + .stopWhenRead(queue.isStopWhenRead()) + .clientId("edqs-" + queue.name().toLowerCase() + "-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName(group)) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-" + queue.name().toLowerCase() + "-producer-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + String requestsConsumerGroup = "edqs-requests-consumer-group-" + edqsConfig.getLabel(); + var requestConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .clientId("edqs-requests-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName(requestsConsumerGroup)) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + var responseProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getResponsesTopic())) + .admin(edqsRequestsAdmin); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer.build()) + .responseTemplate(responseProducer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java new file mode 100644 index 0000000000..e5553078e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLock { + + void lock(); + + void unlock(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java new file mode 100644 index 0000000000..3fe5d7b21b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLockService { + + DistributedLock getLock(String key); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java new file mode 100644 index 0000000000..96483c1f5a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.util.concurrent.locks.ReentrantLock; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +public class DummyDistributedLockService implements DistributedLockService { + + @Override + public DistributedLock getLock(String key) { + return new DummyDistributedLock(); + } + + @RequiredArgsConstructor + private static class DummyDistributedLock implements DistributedLock { + + private final ReentrantLock lock = new ReentrantLock(); + + @SneakyThrows + @Override + public void lock() { + lock.lock(); + } + + @SneakyThrows + @Override + public void unlock() { + lock.unlock(); + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java new file mode 100644 index 0000000000..65405c3d7e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.ZkDiscoveryService; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true") +@Slf4j +public class ZkDistributedLockService implements DistributedLockService { + + private final ZkDiscoveryService zkDiscoveryService; + + @Override + public DistributedLock getLock(String key) { + return new ZkDistributedLock(key); + } + + @RequiredArgsConstructor + private class ZkDistributedLock implements DistributedLock { + + private final InterProcessLock interProcessLock; + + public ZkDistributedLock(String key) { + this.interProcessLock = new InterProcessMutex(zkDiscoveryService.getClient(), zkDiscoveryService.getZkDir() + "/locks/" + key); + } + + @SneakyThrows + @Override + public void lock() { + interProcessLock.acquire(); + } + + @SneakyThrows + @Override + public void unlock() { + interProcessLock.release(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index e774bd74a7..f393a27ddf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -57,7 +57,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { String numPartitionsStr = topicConfigs.get(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); if (numPartitionsStr != null) { numPartitions = Integer.parseInt(numPartitionsStr); - topicConfigs.remove("partitions"); } else { numPartitions = 1; } @@ -71,7 +70,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { return; } try { - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(PropertyUtils.getProps(topicConfigs, properties)); + Map configs = PropertyUtils.getProps(topicConfigs, properties); + configs.remove(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(configs); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { @@ -188,6 +189,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { public boolean isTopicEmpty(String topic) { try { + if (!getTopics().contains(topic)) { + return true; + } TopicDescription topicDescription = settings.getAdminClient().describeTopics(Collections.singletonList(topic)).topicNameValues().get(topic).get(); List partitions = topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())).toList(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java index 3879d2cffd..9d1088e188 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java @@ -26,15 +26,10 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.discovery.PartitionService; import java.time.Duration; import java.util.ArrayList; @@ -56,10 +51,6 @@ public class TbKafkaConsumerStatsService { private final TbKafkaSettings kafkaSettings; private final TbKafkaConsumerStatisticConfig statsConfig; - @Lazy - @Autowired - private PartitionService partitionService; - private Consumer consumer; private ScheduledExecutorService statsPrintScheduler; @@ -111,9 +102,7 @@ public class TbKafkaConsumerStatsService { } private boolean isStatsPrintRequired() { - boolean isMyRuleEnginePartition = partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - boolean isMyCorePartition = partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - return log.isInfoEnabled() && (isMyRuleEnginePartition || isMyCorePartition); + return log.isInfoEnabled(); } private List getTopicsStatsWithLag(Map groupOffsets, Map endOffsets) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 0b86c65b2c..d219428941 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -24,6 +24,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition; import org.springframework.util.StopWatch; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; @@ -33,9 +34,11 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.stream.Collectors; /** @@ -52,7 +55,8 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final String groupId; private final boolean readFromBeginning; // reset offset to beginning - private final boolean stopWhenRead; // stop consuming when reached initial end offsets + private final boolean stopWhenRead; // stop consuming when reached end offset remembered on start + private int readCount; private Map endOffsets; // needed if stopWhenRead is true @Builder @@ -82,29 +86,49 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue } @Override - protected void doSubscribe(List topicNames) { - if (!topicNames.isEmpty()) { - topicNames.forEach(admin::createTopicIfNotExists); - if (readFromBeginning || stopWhenRead) { - consumer.subscribe(topicNames, new ConsumerRebalanceListener() { - @Override - public void onPartitionsRevoked(Collection partitions) {} - - @Override - public void onPartitionsAssigned(Collection partitions) { - log.debug("Handling onPartitionsAssigned {}", partitions); - if (readFromBeginning) { - consumer.seekToBeginning(partitions); - } - if (stopWhenRead) { - endOffsets = consumer.endOffsets(partitions).entrySet().stream() - .filter(entry -> entry.getValue() > 0) - .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); + protected void doSubscribe(Set partitions) { + Map> topics; + if (partitions == null) { + topics = Collections.emptyMap(); + } else { + topics = new HashMap<>(); + partitions.forEach(tpi -> { + if (tpi.isUseInternalPartition()) { + topics.computeIfAbsent(tpi.getFullTopicName(), t -> new ArrayList<>()).add(tpi.getPartition().get()); + } else { + topics.put(tpi.getFullTopicName(), null); + } + }); + } + if (!topics.isEmpty()) { + topics.keySet().forEach(admin::createTopicIfNotExists); + List toSubscribe = new ArrayList<>(); + topics.forEach((topic, kafkaPartitions) -> { + if (kafkaPartitions == null) { + toSubscribe.add(topic); + } else { + List topicPartitions = kafkaPartitions.stream() + .map(partition -> new TopicPartition(topic, partition)) + .toList(); + consumer.assign(topicPartitions); + onPartitionsAssigned(topicPartitions); + } + }); + if (!toSubscribe.isEmpty()) { + if (readFromBeginning || stopWhenRead) { + consumer.subscribe(toSubscribe, new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection partitions) {} + + @Override + public void onPartitionsAssigned(Collection partitions) { + log.debug("Handling onPartitionsAssigned {}", partitions); + TbKafkaConsumerTemplate.this.onPartitionsAssigned(partitions); } - } - }); - } else { - consumer.subscribe(topicNames); + }); + } else { + consumer.subscribe(toSubscribe); + } } } else { log.info("unsubscribe due to empty topic list"); @@ -132,6 +156,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue records.forEach(record -> { recordList.add(record); if (stopWhenRead && endOffsets != null) { + readCount++; int partition = record.partition(); Long endOffset = endOffsets.get(partition); if (endOffset == null) { @@ -146,12 +171,23 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue }); } if (stopWhenRead && endOffsets != null && endOffsets.isEmpty()) { - log.info("Reached end offset for {}, stopping consumer", consumer.assignment()); + log.info("Finished reading {}, processed {} messages", partitions, readCount); stop(); } return recordList; } + private void onPartitionsAssigned(Collection partitions) { + if (readFromBeginning) { + consumer.seekToBeginning(partitions); + } + if (stopWhenRead) { + endOffsets = consumer.endOffsets(partitions).entrySet().stream() + .filter(entry -> entry.getValue() > 0) + .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); + } + } + @Override public T decode(ConsumerRecord record) throws IOException { return decoder.decode(new KafkaTbQueueMsg(record)); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index a4ab0c78d7..cac6f2ea1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -54,7 +54,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro private final TbQueueAdmin admin; - private final Set topics; + private final Set topics; @Getter private final String clientId; @@ -102,14 +102,16 @@ public class TbKafkaProducerTemplate implements TbQueuePro public void send(TopicPartitionInfo tpi, String key, T msg, TbQueueCallback callback) { try { - createTopicIfNotExist(tpi); + String topic = tpi.getFullTopicName(); + createTopicIfNotExist(topic); byte[] data = msg.getData(); ProducerRecord record; List
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); if (log.isDebugEnabled()) { addAnalyticHeaders(headers); } - record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); + Integer partition = tpi.isUseInternalPartition() ? tpi.getPartition().orElse(null) : null; + record = new ProducerRecord<>(topic, partition, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { if (callback != null) { @@ -133,12 +135,12 @@ public class TbKafkaProducerTemplate implements TbQueuePro } } - private void createTopicIfNotExist(TopicPartitionInfo tpi) { - if (topics.contains(tpi)) { + private void createTopicIfNotExist(String topic) { + if (topics.contains(topic)) { return; } - admin.createTopicIfNotExists(tpi.getFullTopicName()); - topics.add(tpi); + admin.createTopicIfNotExists(topic); + topics.add(topic); } @Override @@ -147,4 +149,5 @@ public class TbKafkaProducerTemplate implements TbQueuePro producer.close(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 3cb2d470eb..aebda5a5bc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -56,6 +56,12 @@ public class TbKafkaTopicConfigs { private String calculatedFieldProperties; @Value("${queue.kafka.topic-properties.calculated-field-state:}") private String calculatedFieldStateProperties; + @Value("${queue.kafka.topic-properties.edqs-events:}") + private String edqsEventsProperties; + @Value("${queue.kafka.topic-properties.edqs-requests:}") + private String edqsRequestsProperties; + @Value("${queue.kafka.topic-properties.edqs-state:}") + private String edqsStateProperties; @Getter private Map coreConfigs; @@ -86,7 +92,13 @@ public class TbKafkaTopicConfigs { @Getter private Map calculatedFieldConfigs; @Getter - private Map calculatedFieldStateConfigs; + private Map calculatedFieldStateConfigs; + @Getter + private Map edqsEventsConfigs; + @Getter + private Map edqsRequestsConfigs; + @Getter + private Map edqsStateConfigs; @PostConstruct private void init() { @@ -107,6 +119,9 @@ public class TbKafkaTopicConfigs { edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); calculatedFieldStateConfigs = PropertyUtils.getProps(calculatedFieldStateProperties); + edqsEventsConfigs = PropertyUtils.getProps(edqsEventsProperties); + edqsRequestsConfigs = PropertyUtils.getProps(edqsRequestsProperties); + edqsStateConfigs = PropertyUtils.getProps(edqsStateProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java new file mode 100644 index 0000000000..95be49f82b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; + +public interface EdqsClientQueueFactory { + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 48109279de..e97af10ecc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; @@ -24,13 +25,19 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -45,40 +52,22 @@ import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +@RequiredArgsConstructor public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final TopicService topicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin queueAdmin; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; private final TbQueueCalculatedFieldSettings calculatedFieldSettings; + private final EdqsConfig edqsConfig; private final InMemoryStorage storage; - public InMemoryMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueVersionControlSettings vcSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - TbQueueCalculatedFieldSettings calculatedFieldSettings, - InMemoryStorage storage) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - this.calculatedFieldSettings = calculatedFieldSettings; - this.storage = storage; - } - @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); @@ -244,6 +233,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + TbQueueProducer> requestProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueConsumer> responseConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getResponsesTopic()); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(queueAdmin) + .requestTemplate(requestProducer) + .responseTemplate(responseConsumer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 8791e8c906..bb9c1a028e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -33,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -51,6 +53,8 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -85,6 +89,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueEdgeSettings edgeSettings; private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbKafkaConsumerStatsService consumerStatsService; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -101,6 +106,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; private final TbQueueAdmin cfStateAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); private final AtomicLong edgeConsumerCount = new AtomicLong(); @@ -116,7 +123,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaConsumerStatsService consumerStatsService, - TbKafkaTopicConfigs kafkaTopicConfigs) { + TbKafkaTopicConfigs kafkaTopicConfigs, + EdqsConfig edqsConfig) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -129,6 +137,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; this.calculatedFieldSettings = calculatedFieldSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -145,6 +154,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -573,6 +584,42 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi .build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -609,4 +656,5 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi cfAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 5b0c2743fe..f3adb266a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -31,6 +32,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -49,6 +51,8 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -83,6 +87,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; private final TbQueueCalculatedFieldSettings calculatedFieldSettings; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -98,6 +103,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); private final AtomicLong edgeConsumerCount = new AtomicLong(); @@ -114,6 +121,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, + EdqsConfig edqsConfig, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -127,6 +135,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; this.calculatedFieldSettings = calculatedFieldSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -142,6 +151,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -468,6 +479,42 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -501,4 +548,5 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { cfAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 7167a27d67..43fbb5efeb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -31,6 +32,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -46,6 +48,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -88,6 +91,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; private final TbQueueAdmin cfStateAdmin; + private final TbQueueAdmin edqsEventsAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -121,6 +125,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); } @Override @@ -364,6 +369,20 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + throw new UnsupportedOperationException(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index aba34105ff..037d1f2087 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -44,7 +44,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 920dc5df5b..767fea9f0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -39,7 +39,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index ca34d4006c..6030eb278d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -43,9 +43,8 @@ public class PropertyUtils { } public static Map getProps(Map defaultProperties, String propertiesStr, Function> parser) { - Map properties = defaultProperties; + Map properties = new HashMap<>(defaultProperties); if (StringUtils.isNotBlank(propertiesStr)) { - properties = new HashMap<>(properties); properties.putAll(parser.apply(propertiesStr)); } return properties; diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index d3975c78bd..9e493220ca 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -133,19 +133,19 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenMessages_whenSend_thenOK() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); final int msgCount = 10; for (int i = 0; i < msgCount; i++) { inst.send(getRequestMsgMock()); } assertThat(inst.pendingRequests.mappingCount(), equalTo((long) msgCount)); - verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @Test public void givenMessagesOverMaxPendingRequests_whenSend_thenImmediateFailedFutureForTheOfRequests() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); int msgOverflowCount = 10; for (int i = 0; i < inst.maxPendingRequests; i++) { @@ -155,7 +155,7 @@ public class DefaultTbQueueRequestTemplateTest { assertThat("max pending requests overflow", inst.send(getRequestMsgMock()).isDone(), is(true)); //overflow, immediate failed future } assertThat(inst.pendingRequests.mappingCount(), equalTo(inst.maxPendingRequests)); - verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @SuppressWarnings("unchecked") diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index f626746a64..3680e1bfb2 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -136,7 +136,11 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize); parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize); parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); + parserConfig.registerDataType("TbelCfTsRollingData", TbelCfTsRollingData.class, TbelCfTsRollingData::memorySize); parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); + parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsMultiDoubleVal.class, TbelCfTsMultiDoubleVal::memorySize); + parserConfig.registerDataType("TbelCfCtx", TbelCfCtx.class, TbelCfCtx::memorySize); + TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java index ce61965317..1761ef6d0a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -17,20 +17,24 @@ package org.thingsboard.script.api.tbel; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class TbTimeWindow implements TbelCfObject { public static final long OBJ_SIZE = 32L; private long startTs; private long endTs; - private int limit; @Override public long memorySize() { return OBJ_SIZE; } + public boolean matches(long ts) { + return ts >= startTs && ts < endTs; + } } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 7a54224ddc..ca3a8db02a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -18,6 +18,8 @@ package org.thingsboard.script.api.tbel; import com.google.common.primitives.Bytes; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; +import org.mvel2.ConversionHandler; +import org.mvel2.DataConversion; import org.mvel2.ExecutionContext; import org.mvel2.ParserConfiguration; import org.mvel2.execution.ExecutionArrayList; @@ -255,6 +257,8 @@ public class TbUtils { double.class, int.class))); parserConfig.addImport("toFixed", new MethodStub(TbUtils.class.getMethod("toFixed", float.class, int.class))); + parserConfig.addImport("toInt", new MethodStub(TbUtils.class.getMethod("toInt", + double.class))); parserConfig.addImport("hexToBytes", new MethodStub(TbUtils.class.getMethod("hexToBytes", ExecutionContext.class, String.class))); parserConfig.addImport("hexToBytesArray", new MethodStub(TbUtils.class.getMethod("hexToBytesArray", @@ -1155,6 +1159,10 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(precision, RoundingMode.HALF_UP).floatValue(); } + public static int toInt(double value) { + return BigDecimal.valueOf(value).setScale(0, RoundingMode.HALF_UP).intValue(); + } + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json) { return toFlatMap(ctx, json, new ArrayList<>(), true); } @@ -1506,5 +1514,6 @@ public class TbUtils { } return hex; } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java new file mode 100644 index 0000000000..ce42e2cf3b --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Getter; + +import java.util.Collections; +import java.util.Map; + +public class TbelCfCtx implements TbelCfObject { + + @Getter + private final Map args; + + public TbelCfCtx(Map args) { + this.args = Collections.unmodifiableMap(args); + } + + @Override + public long memorySize() { + return OBJ_SIZE; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java index 3af6198a22..cf575bd9d0 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java @@ -17,6 +17,8 @@ package org.thingsboard.script.api.tbel; public interface TbelCfObject { + long OBJ_SIZE = 32L; // Approximate calculation; + long memorySize(); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java index 193b6ea1ae..84227a8d80 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java @@ -22,8 +22,6 @@ import lombok.Data; @Data public class TbelCfSingleValueArg implements TbelCfArg { - public static final long OBJ_SIZE = 32L; // Approximate calculation; - private final long ts; private final Object value; diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java new file mode 100644 index 0000000000..2743bbd0de --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; + +@Data +public class TbelCfTsMultiDoubleVal implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; // Approximate calculation; + + private final long ts; + private final double[] values; + + @JsonIgnore + public double getV1() { + return getV(0); + } + + @JsonIgnore + public double getV2() { + return getV(1); + } + + @JsonIgnore + public double getV3() { + return getV(2); + } + + @JsonIgnore + public double getV4() { + return getV(3); + } + + @JsonIgnore + public double getV5() { + return getV(4); + } + + private double getV(int idx) { + if (values.length < idx + 1) { + throw new IllegalArgumentException("Can't get value at index " + idx + ". There are " + values.length + " values present."); + } else { + return values[idx]; + } + } + + @Override + public long memorySize() { + return OBJ_SIZE + values.length * 8L; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 807d498a16..3b5a7ac9bd 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -19,11 +19,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; +import org.thingsboard.common.util.JacksonUtil; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeSet; import java.util.function.Consumer; import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; @@ -44,9 +48,9 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable values) { + public TbelCfTsRollingArg(long timeWindow, List values) { long ts = System.currentTimeMillis(); - this.timeWindow = new TbTimeWindow(ts - timeWindow, ts, limit); + this.timeWindow = new TbTimeWindow(ts - timeWindow, ts); this.values = Collections.unmodifiableList(values); } @@ -104,6 +108,14 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable settings) { + return mergeAll(Collections.singletonList(other), settings); + } + + public TbelCfTsRollingData mergeAll(List others) { + return mergeAll(others, null); + } + + public TbelCfTsRollingData mergeAll(List others, Map settings) { + List args = new ArrayList<>(others.size() + 1); + args.add(this); + args.addAll(others); + + boolean ignoreNaN = true; + if (settings != null && settings.containsKey("ignoreNaN")) { + ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); + } + + TbTimeWindow timeWindow = null; + if (settings != null && settings.containsKey("timeWindow")) { + var twVar = settings.get("timeWindow"); + if (twVar instanceof TbTimeWindow) { + timeWindow = (TbTimeWindow) settings.get("timeWindow"); + } else if (twVar instanceof Map twMap) { + timeWindow = new TbTimeWindow(Long.valueOf(twMap.get("startTs").toString()), Long.valueOf(twMap.get("endTs").toString())); + } else { + timeWindow = JacksonUtil.fromString(settings.get("timeWindow").toString(), TbTimeWindow.class); + } + } + + TreeSet allTimestamps = new TreeSet<>(); + long startTs = Long.MAX_VALUE; + long endTs = Long.MIN_VALUE; + for (TbelCfTsRollingArg arg : args) { + for (TbelCfTsDoubleVal val : arg.getValues()) { + allTimestamps.add(val.getTs()); + } + startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); + endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); + } + + List data = new ArrayList<>(); + + int[] lastIndex = new int[args.size()]; + double[] result = new double[args.size()]; + Arrays.fill(result, Double.NaN); + + for (long ts : allTimestamps) { + for (int i = 0; i < args.size(); i++) { + var arg = args.get(i); + var values = arg.getValues(); + while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { + result[i] = values.get(lastIndex[i]).getValue(); + lastIndex[i]++; + } + } + if (timeWindow == null || timeWindow.matches(ts)) { + if (ignoreNaN) { + boolean skip = false; + for (int i = 0; i < args.size(); i++) { + if (Double.isNaN(result[i])) { + skip = true; + break; + } + } + if (!skip) { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } else { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } + } + + return new TbelCfTsRollingData(timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs), data); + } + + @JsonIgnore public int getSize() { return values.size(); @@ -266,11 +360,6 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable action) { - values.forEach(action); - } - @Override public String getType() { return "TS_ROLLING"; diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java new file mode 100644 index 0000000000..646e826915 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; + +import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; + +public class TbelCfTsRollingData implements TbelCfObject, Iterable { + + @Getter + private final TbTimeWindow timeWindow; + @Getter + private final List values; + + public TbelCfTsRollingData(TbTimeWindow timeWindow, List values) { + this.timeWindow = timeWindow; + this.values = Collections.unmodifiableList(values); + } + + @Override + public long memorySize() { + return 12 + values.size() * OBJ_SIZE; + } + + @JsonIgnore + public List getValue() { + return values; + } + + @JsonIgnore + public int getSize() { + return values.size(); + } + + @Override + public Iterator iterator() { + return values.iterator(); + } + +} diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index fbd81948d6..69987d3132 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1109,7 +1109,7 @@ public class TbUtilsTest { String validInput = Base64.getEncoder().encodeToString(new byte[]{1, 2, 3, 4, 5}); ExecutionArrayList actual = TbUtils.base64ToBytesList(ctx, validInput); ExecutionArrayList expected = new ExecutionArrayList<>(ctx); - expected.addAll(List.of((byte) 1, (byte)2, (byte)3, (byte)4, (byte)5)); + expected.addAll(List.of((byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5)); Assertions.assertEquals(expected, actual); String emptyInput = Base64.getEncoder().encodeToString(new byte[]{}); @@ -1123,6 +1123,7 @@ public class TbUtilsTest { TbUtils.base64ToBytesList(ctx, null); }); } + @Test public void bytesToHex_Test() { byte[] bb = {(byte) 0xBB, (byte) 0xAA}; @@ -1136,6 +1137,13 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } + @Test + void toInt() { + Assertions.assertEquals(1729, TbUtils.toInt(doubleVal)); + Assertions.assertEquals(13, TbUtils.toInt(12.8)); + Assertions.assertEquals(28, TbUtils.toInt(28.0)); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java index 327b477b6f..69eba2fab2 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java @@ -15,10 +15,15 @@ */ package org.thingsboard.script.api.tbel; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.JacksonUtil; +import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -33,7 +38,7 @@ public class TbelCfTsRollingArgTest { @BeforeEach void setUp() { rollingArg = new TbelCfTsRollingArg( - new TbTimeWindow(ts - 30000, ts - 10, 10), + new TbTimeWindow(ts - 30000, ts - 10), List.of( new TbelCfTsDoubleVal(ts - 10, Double.NaN), new TbelCfTsDoubleVal(ts - 20, 2.0), @@ -98,7 +103,7 @@ public class TbelCfTsRollingArgTest { void testFirstAndLastWhenOnlyNaNAndIgnoreNaNIsFalse() { assertThat(rollingArg.first()).isEqualTo(2.0); rollingArg = new TbelCfTsRollingArg( - new TbTimeWindow(ts - 30000, ts - 10, 10), + new TbTimeWindow(ts - 30000, ts - 10), List.of( new TbelCfTsDoubleVal(ts - 10, Double.NaN), new TbelCfTsDoubleVal(ts - 40, Double.NaN), @@ -117,7 +122,7 @@ public class TbelCfTsRollingArgTest { @Test void testEmptyValues() { - rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10, 10), List.of()); + rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10), List.of()); assertThatThrownBy(rollingArg::sum).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); assertThatThrownBy(rollingArg::max).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); assertThatThrownBy(rollingArg::min).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); @@ -128,4 +133,81 @@ public class TbelCfTsRollingArgTest { assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); } + @Test + public void merge_two_rolling_args_ts_match_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_with_timewindow_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("timeWindow", new TbTimeWindow(0, 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + + result = arg1.merge(arg2, Collections.singletonMap("timeWindow", Map.of("startTs", 0L, "endTs", 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_default_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(200L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(11, item0.getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("ignoreNaN", false)); + Assertions.assertEquals(4, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(100L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(Double.NaN, item0.getValues()[1]); + + TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); + Assertions.assertEquals(200L, item1.getTs()); + Assertions.assertEquals(1, item1.getValues()[0]); + Assertions.assertEquals(11, item1.getValues()[1]); + } + } \ No newline at end of file diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index a21b52eaf4..de64d4fe54 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -22,7 +22,8 @@ public enum StatsType { JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"), HOUSEKEEPER("housekeeper"), - EDGE("edge"); + EDGE("edge"), + EDQS("edqs"); private final String name; diff --git a/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java b/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java index 008158246e..0f1a56cb17 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java @@ -36,6 +36,9 @@ public class DonAsynchron { FutureCallback callback = new FutureCallback() { @Override public void onSuccess(T result) { + if (onSuccess == null) { + return; + } try { onSuccess.accept(result); } catch (Throwable th) { @@ -45,6 +48,9 @@ public class DonAsynchron { @Override public void onFailure(Throwable t) { + if (onFailure == null) { + return; + } onFailure.accept(t); } }; diff --git a/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java b/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java index e08a8de30a..c93ec09bbf 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java @@ -64,4 +64,14 @@ public class ExceptionUtil { } } } + + public static String getMessage(Throwable t) { + String message = t.getMessage(); + if (StringUtils.isNotEmpty(message)) { + return message; + } else { + return t.getClass().getSimpleName(); + } + } + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 737de124c8..0f8969f9a4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -175,7 +175,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } } } - consumer.subscribe(event.getPartitionsMap().values().stream().findAny().orElse(Collections.emptySet())); + consumer.subscribe(event.getNewPartitions().values().stream().findAny().orElse(Collections.emptySet())); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index a96e401869..72883c55ef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.id.TenantId; import java.util.Collection; @@ -45,6 +46,12 @@ public interface Dao { List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit); - default EntityType getEntityType() { return null; } + default List findNextBatch(UUID id, int batchSize) { + throw new UnsupportedOperationException(); + } + + default EntityType getEntityType() { + return null; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index c9dcc87fa9..43b651bb24 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.dao.model.ToData; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -85,6 +86,10 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } + public static Pageable toPageable(PageLink pageLink, String... sortColumns) { + return toPageable(pageLink, Collections.emptyMap(), Arrays.stream(sortColumns).map(column -> new SortOrder(column, SortOrder.Direction.ASC)).toList(), false); + } + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return toPageable(pageLink, columnMap, sortOrders, true); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index 1ebf148b1c..c295108a47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -16,8 +16,17 @@ package org.thingsboard.server.dao; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; -public interface TenantEntityDao { +public interface TenantEntityDao { + + default Long countByTenantId(TenantId tenantId) { + throw new UnsupportedOperationException(); + } + + default PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + throw new UnsupportedOperationException(); + } - Long countByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index d246dfaf8b..36700ff59f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -37,7 +37,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. @@ -239,4 +239,5 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD PageData> getAllAssetTypes(PageLink pageLink); PageData findProfileEntityIdInfos(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 3ba9160975..6bf44e4da1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -148,12 +148,11 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); + List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize); + List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 6e3759a991..777a77d054 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; @@ -24,13 +26,18 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.Validator; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -45,16 +52,15 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true) @Primary @Slf4j +@RequiredArgsConstructor public class BaseAttributesService implements AttributesService { + private final AttributesDao attributesDao; + private final EdqsService edqsService; @Value("${sql.attributes.value_no_xss_validation:false}") private boolean valueNoXssValidation; - public BaseAttributesService(AttributesDao attributesDao) { - this.attributesDao = attributesDao; - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -98,26 +104,53 @@ public class BaseAttributesService implements AttributesService { public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, attribute); } @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> doSave(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + return Futures.transform(future, version -> { + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); + return version; + }, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); + List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); + return Futures.transform(Futures.allAsList(futures), result -> { + List keys = new ArrayList<>(); + for (TbPair keyVersionPair : result) { + String key = keyVersionPair.getFirst(); + Long version = keyVersionPair.getSecond(); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } + keys.add(key); + } + return keys; + }, MoreExecutors.directExecutor()); } @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + deleted.forEach(attribute -> { + AttributeScope scope = attribute.getKey(); + String key = attribute.getValue(); + if (scope != null && key != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); + } + }); return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index d9afa69ba7..559828911f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -24,18 +24,22 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; @@ -67,6 +71,7 @@ public class CachedAttributesService implements AttributesService { private final AttributesDao attributesDao; private final JpaExecutorService jpaExecutorService; private final CacheExecutorService cacheExecutorService; + private final EdqsService edqsService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; private final VersionedTbCache cache; @@ -79,11 +84,12 @@ public class CachedAttributesService implements AttributesService { public CachedAttributesService(AttributesDao attributesDao, JpaExecutorService jpaExecutorService, - StatsFactory statsFactory, + @Lazy EdqsService edqsService, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, VersionedTbCache cache) { this.attributesDao = attributesDao; this.jpaExecutorService = jpaExecutorService; + this.edqsService = edqsService; this.cacheExecutorService = cacheExecutorService; this.cache = cache; @@ -237,8 +243,10 @@ public class CachedAttributesService implements AttributesService { private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - put(entityId, scope, new BaseAttributeKvEntry(((BaseAttributeKvEntry)attribute).getKv(), attribute.getLastUpdateTs(), version)); + return Futures.transform(future, version -> { + BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); + put(entityId, scope, attributeKvEntry); + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); return version; }, cacheExecutor); } @@ -256,7 +264,11 @@ public class CachedAttributesService implements AttributesService { List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, keyVersionPair -> { String key = keyVersionPair.getFirst(); - cache.evict(new AttributeCacheKey(scope, entityId, key), keyVersionPair.getSecond()); + Long version = keyVersionPair.getSecond(); + cache.evict(new AttributeCacheKey(scope, entityId, key), version); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } return key; }, cacheExecutor)).collect(Collectors.toList())); } @@ -269,6 +281,8 @@ public class CachedAttributesService implements AttributesService { String key = deleted.getValue(); if (scope != null && key != null) { cache.evict(new AttributeCacheKey(scope, entityId, key)); + // using version as Long.MAX_VALUE because we expect that the entity is deleted and there won't be any attributes after this + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); } }); return result.size(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 416bff6ea3..dceef73aba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.cf; -import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java index 5c24c45c16..258bac8dbd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java @@ -29,6 +29,7 @@ import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.dao.model.sql.AuditLogEntity; +import org.thingsboard.server.dao.model.sql.CalculatedFieldDebugEventEntity; import org.thingsboard.server.dao.model.sql.ErrorEventEntity; import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity; @@ -68,7 +69,7 @@ public class DedicatedEventsJpaDaoConfig { EntityManagerFactoryBuilder builder) { return builder .dataSource(eventsDataSource) - .packages(LifecycleEventEntity.class, StatisticsEventEntity.class, ErrorEventEntity.class, RuleNodeDebugEventEntity.class, RuleChainDebugEventEntity.class, AuditLogEntity.class) + .packages(LifecycleEventEntity.class, StatisticsEventEntity.class, ErrorEventEntity.class, RuleNodeDebugEventEntity.class, RuleChainDebugEventEntity.class, AuditLogEntity.class, CalculatedFieldDebugEventEntity.class) .persistenceUnit(EVENTS_PERSISTENCE_UNIT) .build(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index b0a23bf6fc..c6bfa970a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 562143be36..4b05cdbc75 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -162,7 +162,7 @@ public class CustomerServiceImpl extends AbstractCachedEntityService, TenantEntityDao, ExportableEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index b8fff17a50..1b21310237 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -172,7 +172,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb var saved = dashboardDao.save(tenantId, dashboard); publishEvictEvent(new DashboardTitleEvictEvent(saved.getId())); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId) - .entityId(saved.getId()).created(dashboard.getId() == null).build()); + .entityId(saved.getId()).entity(saved).created(dashboard.getId() == null).build()); if (dashboard.getId() == null) { countService.publishCountEntityEvictEvent(tenantId, EntityType.DASHBOARD); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index f60da9a603..efc57119eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -42,7 +42,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 61523fd626..2ebcf046a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -180,13 +180,12 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService findAll(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java index c6d456c2d9..fdb9144ab2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java @@ -35,7 +35,7 @@ import java.util.UUID; * The Interface EdgeDao. * */ -public interface EdgeDao extends Dao, TenantEntityDao { +public interface EdgeDao extends Dao, TenantEntityDao { Edge save(TenantId tenantId, Edge edge); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 07a704b8e5..a762aabf38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.HasLabel; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTitle; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -40,8 +42,10 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import java.util.ArrayList; @@ -50,10 +54,12 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.id.EntityId.NULL_UUID; +import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_TYPE; import static org.thingsboard.server.dao.service.Validator.validateEntityDataPageLink; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -79,12 +85,24 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe @Lazy EntityServiceRegistry entityServiceRegistry; + @Autowired + @Lazy + private EdqsApiService edqsApiService; + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); + + if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { + EdqsRequest request = EdqsRequest.builder() + .entityCountQuery(query) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityCountQueryResult(); + } return this.entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } @@ -95,6 +113,14 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); + if (edqsApiService.isEnabled() && validForEdqs(query)) { + EdqsRequest request = EdqsRequest.builder() + .entityDataQuery(query) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityDataQueryResult(); + } + if (!isValidForOptimization(query)) { return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); } @@ -110,6 +136,25 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return new PageData<>(result, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); } + private boolean validForEdqs(EntityCountQuery query) { // for compatibility with PE + return true; + } + + private EdqsResponse processEdqsRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + EdqsResponse response; + try { + log.debug("[{}] Sending request to EDQS: {}", tenantId, request); + response = edqsApiService.processRequest(tenantId, customerId, request).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + log.debug("[{}] Received response from EDQS: {}", tenantId, response); + if (response.getError() != null) { + throw new RuntimeException(response.getError()); + } + return response; + } + @Override public Optional fetchEntityName(TenantId tenantId, EntityId entityId) { log.trace("Executing fetchEntityName [{}]", entityId); @@ -189,6 +234,8 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe throw new IncorrectParameterException("Query entity filter type must be specified."); } else if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) { validateRelationQuery((RelationsQueryFilter) query.getEntityFilter()); + } else if (query.getEntityFilter().getType().equals(ENTITY_TYPE)) { + validateEntityTypeQuery((EntityTypeFilter) query.getEntityFilter()); } } @@ -197,6 +244,12 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateEntityDataPageLink(query.getPageLink()); } + private static void validateEntityTypeQuery(EntityTypeFilter filter) { + if (filter.getEntityType() == null) { + throw new IncorrectParameterException("Entity type is required"); + } + } + private static void validateRelationQuery(RelationsQueryFilter queryFilter) { if (queryFilter.isMultiRoot() && queryFilter.getMultiRootEntitiesType() == null) { throw new IncorrectParameterException("Multi-root relation query filter should contain 'multiRootEntitiesType'"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 74deb29247..2dbda352b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -26,6 +26,7 @@ import java.util.Map; @Service @Slf4j +@SuppressWarnings({"unchecked"}) public class EntityDaoRegistry { private final Map> daos = new EnumMap<>(EntityType.class); @@ -39,7 +40,6 @@ public class EntityDaoRegistry { }); } - @SuppressWarnings("unchecked") public Dao getDao(EntityType entityType) { Dao dao = (Dao) daos.get(entityType); if (dao == null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 867264cfaf..0e742e20db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -123,7 +123,7 @@ public class EntityViewServiceImpl extends CachedVersionedEntityService { private final T oldEntity; private final EntityId entityId; private final Boolean created; + private final Boolean broadcastEvent; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index e2b21ed59b..148908d063 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -729,7 +729,6 @@ public class ModelConstants { public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; public static final String CALCULATED_FIELD_VERSION = "version"; - public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; /** * Calculated field links constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 349091a1ae..de6a1365b1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -40,7 +40,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_C import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_TYPE; -import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_EXTERNAL_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_NAME; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; @@ -82,9 +81,6 @@ public class CalculatedFieldEntity extends BaseVersionedEntity @Column(name = DEBUG_SETTINGS) private String debugSettings; - @Column(name = CALCULATED_FIELD_EXTERNAL_ID) - private UUID externalId; - public CalculatedFieldEntity() { super(); } @@ -101,9 +97,6 @@ public class CalculatedFieldEntity extends BaseVersionedEntity this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); - if (calculatedField.getExternalId() != null) { - this.externalId = calculatedField.getExternalId().getId(); - } } @Override @@ -118,9 +111,6 @@ public class CalculatedFieldEntity extends BaseVersionedEntity calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); - if (externalId != null) { - calculatedField.setExternalId(new CalculatedFieldId(externalId)); - } return calculatedField; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java index 782872b12f..b3870a9e3a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java @@ -26,6 +26,7 @@ import jakarta.persistence.SqlResultSetMapping; import jakarta.persistence.SqlResultSetMappings; import jakarta.persistence.Table; import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; @@ -91,4 +92,12 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity { this.strKey = strKey; this.version = version; } + + @Override + public TsKvEntry toData() { + TsKvEntry tsKvEntry = super.toData(); + tsKvEntry.setVersion(version); + return tsKvEntry; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java index 7505d314d4..eeae618c61 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { PageData findByTenantIdAndPageLink(TenantId tenantId, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index bc072567d6..f8f877e55e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,5 +21,7 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityWithDataDao; public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { + Long sumDataSizeByTenantId(TenantId tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index 87c2b513a2..9e4c7136d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -51,7 +53,10 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public QueueStats save(TenantId tenantId, QueueStats queueStats) { log.trace("Executing save [{}]", queueStats); queueStatsValidator.validate(queueStats, QueueStats::getTenantId); - return queueStatsDao.save(tenantId, queueStats); + QueueStats savedQueueStats = queueStatsDao.save(tenantId, queueStats); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedQueueStats.getTenantId()).entityId(savedQueueStats.getId()) + .entity(savedQueueStats).created(queueStats.getId() == null).build()); + return savedQueueStats; } @Override @@ -80,7 +85,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public PageData findByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findByTenantId, tenantId: [{}]", tenantId); Validator.validatePageLink(pageLink); - return queueStatsDao.findByTenantId(tenantId, pageLink); + return queueStatsDao.findAllByTenantId(tenantId, pageLink); } @Override @@ -93,6 +98,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu @Override public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { queueStatsDao.removeById(tenantId, id.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java index c9b6df0016..cd1f0701fe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java @@ -17,19 +17,16 @@ package org.thingsboard.server.dao.queue; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface QueueStatsDao extends Dao { +public interface QueueStatsDao extends Dao, TenantEntityDao { QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String queueName, String serviceId); - PageData findByTenantId(TenantId tenantId, PageLink pageLink); - void deleteByTenantId(TenantId tenantId); List findByIds(TenantId tenantId, List queueStatsIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index b5fde88a0b..bea9e6e7e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -133,10 +133,9 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC if (ruleChain.getId() == null) { entityCountService.publishCountEntityEvictEvent(ruleChain.getTenantId(), EntityType.RULE_CHAIN); } - if (publishSaveEvent) { - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedRuleChain.getTenantId()) - .entity(savedRuleChain).entityId(savedRuleChain.getId()).created(ruleChain.getId() == null).build()); - } + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedRuleChain.getTenantId()) + .entity(savedRuleChain).entityId(savedRuleChain.getId()).created(ruleChain.getId() == null) + .broadcastEvent(publishSaveEvent).build()); return savedRuleChain; } catch (Exception e) { checkConstraintViolation(e, "rule_chain_external_id_unq_key", "Rule Chain with such external id already exists!"); @@ -298,9 +297,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC relationService.saveRelations(tenantId, relations); } ruleChain = ruleChainDao.save(tenantId, ruleChain); - if (publishSaveEvent) { - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId).entity(ruleChain).entityId(ruleChain.getId()).build()); - } + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId).entity(ruleChain) + .entityId(ruleChain.getId()).broadcastEvent(publishSaveEvent).build()); return RuleChainUpdateResult.successful(updatedRuleNodes); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index f5bc8a9c56..5b09eec42a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -31,7 +31,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java index 93e2b0f2c3..1a24ff6abb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java @@ -35,5 +35,9 @@ public interface AlarmCommentRepository extends JpaRepository findAllByAlarmId(@Param("alarmId") UUID alarmId, - Pageable pageable); + Pageable pageable); + + @Query("SELECT c FROM AlarmCommentEntity c WHERE c.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 9cfa7a63eb..b9c1abe416 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -414,4 +414,6 @@ public interface AlarmRepository extends JpaRepository { @Param("alarmSeverities") List alarmSeverities, int limit); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index fa97fe258e..51eeb1c21d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.alarm; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -42,4 +44,6 @@ public interface EntityAlarmRepository extends JpaRepository findAllByEntityId(UUID entityId); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java index 0fb7220784..42c9d68b01 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmCommentDao; import org.thingsboard.server.dao.model.sql.AlarmCommentEntity; import org.thingsboard.server.dao.sql.JpaPartitionedAbstractDao; @@ -44,7 +45,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ALARM_COMMENT_TABL @Component @SqlDao @RequiredArgsConstructor -public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao { +public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao, TenantEntityDao { private final SqlPartitioningRepository partitioningRepository; @Value("${sql.alarm_comments.partition_size:168}") private int partitionSizeInHours; @@ -76,6 +77,11 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmCommentRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override protected Class getEntityClass() { return AlarmCommentEntity.class; @@ -85,4 +91,5 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao getRepository() { return alarmCommentRepository; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 34b8fc91e1..c21d8ae928 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -57,6 +57,7 @@ import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.OriginatorAlarmFilter; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AlarmEntity; @@ -84,7 +85,7 @@ import static org.thingsboard.server.dao.DaoUtil.toPageable; @Slf4j @Component @SqlDao -public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao { +public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao, TenantEntityDao { @Autowired private AlarmRepository alarmRepository; @@ -551,6 +552,11 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ALARM; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java new file mode 100644 index 0000000000..796fe63da8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.alarm.EntityAlarm; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.util.SqlDao; + +@Component +@SqlDao +public class JpaEntityAlarmDao implements TenantEntityDao { + + @Autowired + private EntityAlarmRepository entityAlarmRepository; + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(entityAlarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "entityId", "alarmId"))); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java index fbcfbbda8a..eb35a4e18e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.asset; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,6 +23,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; @@ -81,4 +83,8 @@ public interface AssetProfileRepository extends JpaRepository findAllTenantAssetProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetProfileFields(a.id, a.createdTime, a.tenantId," + + "a.name, a.version, a.isDefault) FROM AssetProfileEntity a WHERE a.id > :id ORDER BY a.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 5600f7f7d0..e475864684 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -15,12 +15,13 @@ */ package org.thingsboard.server.dao.sql.asset; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; @@ -226,4 +227,9 @@ public interface AssetRepository extends JpaRepository, Expor @Query(value = "SELECT DISTINCT new org.thingsboard.server.common.data.util.TbPair(a.tenantId , a.type) FROM AssetEntity a") Page> getAllAssetTypes(Pageable pageable); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetFields(a.id, a.createdTime, a.tenantId, a.customerId," + + "a.name, a.version, a.type, a.label, a.assetProfileId, a.additionalInfo) FROM AssetEntity a WHERE a.id > :id ORDER BY a.id") + List findAllFields(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index bb42abe4da..c61d894de5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.asset; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; @@ -25,6 +26,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -290,6 +292,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A .map(AssetId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID uuid, int batchSize) { + return assetRepository.findAllFields(uuid, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java index c64efe01c7..eeab5a338a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.asset; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; @@ -23,11 +24,13 @@ import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.asset.AssetProfileDao; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -37,7 +40,7 @@ import java.util.Optional; import java.util.UUID; @Component -public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao { +public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao, TenantEntityDao { @Autowired private AssetProfileRepository assetProfileRepository; @@ -138,6 +141,16 @@ public class JpaAssetProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return assetProfileRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.ASSET_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index f016de2878..e06975a9a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -59,4 +59,12 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByEntityIdsAndAttributeType(@Param("entityIds") List entityIds, @Param("attributeType") int attributeType); + + @Query(value = "SELECT attribute_key, attribute_type, entity_id, bool_v, dbl_v, json_v, last_update_ts, long_v, str_v, version FROM attribute_kv WHERE (entity_id, attribute_type, attribute_key) > " + + "(:entityId, :attributeType, :attributeKey) ORDER BY entity_id, attribute_type, attribute_key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("attributeType") int attributeType, + @Param("attributeKey") int attributeKey, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 9d477ee5b3..0a8b8f6399 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -49,6 +49,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -152,6 +153,11 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return DaoUtil.convertDataList(Lists.newArrayList(attributes)); } + @Override + public List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize) { + return attributeKvRepository.findNextBatch(entityId, attributeType, attributeKey, batchSize); + } + @Override public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { if (deviceProfileId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fae3468f1e..e59ff3f4e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -92,7 +92,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); - calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); return calculatedField; }).collect(Collectors.toList()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index 2fd98c8bdf..dbb2fd87da 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.sql.cf; -import com.google.common.util.concurrent.ListenableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index af4c96fa63..8ad7311423 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.dao.sql.customer; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CustomerEntity; +import java.util.List; import java.util.UUID; /** @@ -55,4 +58,8 @@ public interface CustomerRepository extends JpaRepository, nativeQuery = true) Page findCustomersWithTheSameTitle(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.CustomerFields(c.id, c.createdTime, c.tenantId, " + + "c.title, c.version, c.additionalInfo, c.country, c.state, c.city, c.address, c.address2, c.zip, c.phone, c.email) " + + "FROM CustomerEntity c WHERE c.id > :id ORDER BY c.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 4c3d0083a6..75e7179391 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.sql.customer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -30,6 +32,7 @@ import org.thingsboard.server.dao.model.sql.CustomerEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -104,6 +107,16 @@ public class JpaCustomerDao extends JpaAbstractDao imp ); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return customerRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 94b6cd541c..f4e934e64b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.dashboard; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -46,4 +48,7 @@ public interface DashboardRepository extends JpaRepository findAllIds(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DashboardFields(d.id, d.createdTime, d.tenantId, " + + "d.assignedCustomers, d.title, d.version) FROM DashboardEntity d WHERE d.id > :id ORDER BY d.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index ea78114ead..2d796c6917 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.sql.dashboard; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -90,6 +92,16 @@ public class JpaDashboardDao extends JpaAbstractDao return DaoUtil.pageToPageData(dashboardRepository.findAllIds(DaoUtil.toPageable(pageLink)).map(DashboardId::new)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return dashboardRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index 2f84d24420..7b938ea274 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -36,4 +38,8 @@ public interface DeviceCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index 34dac8c7a4..88d4780f9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -23,6 +24,7 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; @@ -92,4 +94,7 @@ public interface DeviceProfileRepository extends JpaRepository findAllTenantDeviceProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields(d.id, d.createdTime, d.tenantId," + + "d.name, d.version, d.type, d.isDefault) FROM DeviceProfileEntity d WHERE d.id > :id ORDER BY d.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index 1527801e1c..f4c2fed9fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -202,4 +204,9 @@ public interface DeviceRepository extends JpaRepository, Exp @Query("SELECT externalId FROM DeviceEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceFields(d.id, d.createdTime, d.tenantId, d.customerId," + + "d.name, d.version, d.type, d.label, d.deviceProfileId, d.additionalInfo) FROM DeviceEntity d WHERE d.id > :id ORDER BY d.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 6d9093dc0b..7445d058d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -21,8 +21,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceCredentialsDao; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -36,7 +39,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao { +public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao, TenantEntityDao { @Autowired private DeviceCredentialsRepository deviceCredentialsRepository; @@ -67,4 +70,9 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(deviceCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index aa5069c4b5..34835f52f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.device; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -300,6 +302,16 @@ public class JpaDeviceDao extends JpaAbstractDao implement .map(DeviceId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return deviceRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index d68033d14b..ebd8c78ed0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; @@ -25,11 +26,13 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -41,7 +44,7 @@ import java.util.UUID; @Component @SqlDao -public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao { +public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao, TenantEntityDao { @Autowired private DeviceProfileRepository deviceProfileRepository; @@ -156,6 +159,16 @@ public class JpaDeviceProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findDeviceProfiles(tenantId, pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return deviceProfileRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java index e1c1c71708..42f6b4c819 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java @@ -15,6 +15,4 @@ */ package org.thingsboard.server.dao.sql.device; -public interface NativeAssetRepository extends NativeProfileEntityRepository { - -} +public interface NativeAssetRepository extends NativeProfileEntityRepository {} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java index c9c783cc46..b9fcf75ba4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java @@ -24,5 +24,4 @@ public interface NativeDeviceRepository extends NativeProfileEntityRepository { PageData findDeviceIdInfos(Pageable pageable); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java index 245b47ae63..c11db0a348 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.edge; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.dao.model.sql.EdgeEntity; import org.thingsboard.server.dao.model.sql.EdgeInfoEntity; @@ -154,4 +156,7 @@ public interface EdgeRepository extends JpaRepository { EdgeEntity findByRoutingKey(String routingKey); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.EdgeFields(e.id, e.createdTime, e.tenantId, e.customerId," + + "e.name, e.version, e.type, e.label, e.additionalInfo) FROM EdgeEntity e WHERE e.id > :id ORDER BY e.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index b1c51c73c1..3f45b1ca1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -18,12 +18,14 @@ package org.thingsboard.server.dao.sql.edge; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -219,6 +221,16 @@ public class JpaEdgeDao extends JpaAbstractDao implements Edge return edgeRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findEdgesByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return edgeRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.EDGE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 0112be4967..71d2f04b81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.entityview; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.EntityViewFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -145,4 +147,7 @@ public interface EntityViewRepository extends JpaRepository :id ORDER BY e.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index a6c4457f4f..44d8a09ff4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -18,17 +18,20 @@ package org.thingsboard.server.dao.sql.entityview; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.edqs.fields.EntityViewFields; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -47,8 +50,7 @@ import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityTypesToDto; @Component @Slf4j @SqlDao -public class JpaEntityViewDao extends JpaAbstractDao - implements EntityViewDao { +public class JpaEntityViewDao extends JpaAbstractDao implements EntityViewDao, TenantEntityDao { @Autowired private EntityViewRepository entityViewRepository; @@ -218,8 +220,19 @@ public class JpaEntityViewDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return entityViewRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.ENTITY_VIEW; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java index 6781dda593..739423ee70 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.config.Notif import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.NotificationRuleEntity; import org.thingsboard.server.dao.model.sql.NotificationRuleInfoEntity; import org.thingsboard.server.dao.notification.NotificationRuleDao; @@ -41,7 +42,7 @@ import java.util.UUID; @Component @SqlDao @RequiredArgsConstructor -public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao { +public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao, TenantEntityDao { private final NotificationRuleRepository notificationRuleRepository; @@ -101,6 +102,11 @@ public class JpaNotificationRuleDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationRuleEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java index 07799d2388..1efce2de01 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java @@ -101,6 +101,11 @@ public class JpaNotificationTargetDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationTargetEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java index 8941bf119d..0f7d6e1d64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.notification.template.NotificationTemp import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.NotificationTemplateEntity; import org.thingsboard.server.dao.notification.NotificationTemplateDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +39,7 @@ import java.util.UUID; @Component @SqlDao @RequiredArgsConstructor -public class JpaNotificationTemplateDao extends JpaAbstractDao implements NotificationTemplateDao { +public class JpaNotificationTemplateDao extends JpaAbstractDao implements NotificationTemplateDao, TenantEntityDao { private final NotificationTemplateRepository notificationTemplateRepository; @@ -83,6 +84,11 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected JpaRepository getRepository() { return notificationTemplateRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java index 24ba852cd8..780f67932e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java @@ -19,9 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.OtaPackageEntity; import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -32,7 +37,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaOtaPackageDao extends JpaAbstractDao implements OtaPackageDao { +public class JpaOtaPackageDao extends JpaAbstractDao implements OtaPackageDao, TenantEntityDao { @Autowired private OtaPackageRepository otaPackageRepository; @@ -52,6 +57,12 @@ public class JpaOtaPackageDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(otaPackageRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.OTA_PACKAGE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java index dec6d36ec6..208b296d1c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java @@ -93,4 +93,5 @@ public class JpaOtaPackageInfoDao extends JpaAbstractDao { + @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); + + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index c990a2dba4..9e20a54b14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataPageLink; @@ -44,7 +45,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.UUID; import java.util.stream.Collectors; @Repository @@ -129,7 +129,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { public PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds) { return transactionTemplate.execute(trStatus -> { AlarmDataPageLink pageLink = query.getPageLink(); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); ctx.addUuidListParameter("entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); StringBuilder selectPart = new StringBuilder(FIELDS_SELECTION); StringBuilder fromPart = new StringBuilder(" from alarm_info a "); @@ -316,7 +316,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { @Override public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); if (query.isSearchPropagatedAlarms()) { ctx.append("select count(distinct(a.id)) from alarm_info a "); @@ -419,7 +419,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { }); } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && selectionMapping != null && !selectionMapping.isEmpty()) { String lowerSearchText = searchText.toLowerCase() + "%"; List searchPredicates = selectionMapping.stream() @@ -437,7 +437,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } } - private String buildPermissionsQuery(TenantId tenantId, QueryContext ctx) { + private String buildPermissionsQuery(TenantId tenantId, SqlQueryContext ctx) { StringBuilder permissionsQuery = new StringBuilder(); ctx.addUuidParameter("permissions_tenant_id", tenantId.getId()); permissionsQuery.append(" a.tenant_id = :permissions_tenant_id and ea.tenant_id = :permissions_tenant_id "); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index ef712fa286..8f3fe71f35 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; import org.thingsboard.server.common.data.query.AssetTypeFilter; @@ -334,7 +335,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); if (query.getKeyFilters() == null || query.getKeyFilters().isEmpty()) { ctx.append("select count(e.id) from "); ctx.append(addEntityTableQuery(ctx, query.getEntityFilter())); @@ -416,7 +417,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck) { return transactionTemplate.execute(status -> { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, ignorePermissionCheck)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, ignorePermissionCheck)); EntityDataPageLink pageLink = query.getPageLink(); List mappings = EntityKeyMapping.prepareKeyMapping(entityType, query); @@ -524,7 +525,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { }); } - private String buildEntityWhere(QueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { + private String buildEntityWhere(SqlQueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { String permissionQuery = this.buildPermissionQuery(ctx, entityFilter); String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, entityFieldsFilters, entityFilter.getType()); @@ -538,7 +539,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return result; } - private String buildPermissionQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildPermissionQuery(SqlQueryContext ctx, EntityFilter entityFilter) { if (ctx.isIgnorePermissionCheck()) { return "1=1"; } @@ -575,7 +576,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String defaultPermissionQuery(QueryContext ctx) { + private String defaultPermissionQuery(SqlQueryContext ctx) { ctx.addUuidParameter("permissions_tenant_id", ctx.getTenantId().getId()); if (ctx.getCustomerId() != null && !ctx.getCustomerId().isNullUid()) { ctx.addUuidParameter("permissions_customer_id", ctx.getCustomerId().getId()); @@ -593,7 +594,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String buildEntityFilterQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildEntityFilterQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case SINGLE_ENTITY: return this.singleEntityQuery(ctx, (SingleEntityFilter) entityFilter); @@ -619,7 +620,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String addEntityTableQuery(QueryContext ctx, EntityFilter entityFilter) { + private String addEntityTableQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case RELATIONS_QUERY: return relationQuery(ctx, (RelationsQueryFilter) entityFilter); @@ -640,7 +641,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String entitySearchQuery(QueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { + private String entitySearchQuery(SqlQueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, additional_info " @@ -680,7 +681,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return query; } - private String relationQuery(QueryContext ctx, RelationsQueryFilter entityFilter) { + private String relationQuery(SqlQueryContext ctx, RelationsQueryFilter entityFilter) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = SELECT_TENANT_ID + ", " + SELECT_CUSTOMER_ID @@ -692,6 +693,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE + ", " + SELECT_ADDITIONAL_INFO + (entityFilter.isMultiRoot() ? (", " + SELECT_RELATED_PARENT_ID) : "") + ", entity.entity_type as entity_type"; + /* + * FIXME: + * target entities are duplicated in result list, if search direction is TO and multiple relations are references to target entity + * */ String from = getQueryTemplate(entityFilter.getDirection(), entityFilter.isMultiRoot()); if (entityFilter.isMultiRoot()) { @@ -763,7 +768,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return "( " + selectFields + from + ")"; } - private String buildEtfCondition(QueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { + private String buildEtfCondition(SqlQueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { StringBuilder whereFilter = new StringBuilder(); String relationType = etf.getRelationType(); List entityTypes = etf.getEntityTypes(); @@ -812,7 +817,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return from; } - private String buildAliasWhereQuery(QueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { + private String buildAliasWhereQuery(SqlQueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { List aliasFiltersMapping = selectionMapping.stream().filter(mapping -> !mapping.isLatest() && mapping.getEntityKeyColumn() == null) .collect(Collectors.toList()); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, aliasFiltersMapping, entityFilter.getType()); @@ -822,12 +827,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { result += " where (" + entityFieldsQuery + ")"; } if (!searchTextQuery.isEmpty()) { - result += (result.isEmpty() ? " where ": " and ") + "(" + searchTextQuery + ") "; + result += (result.isEmpty() ? " where " : " and ") + "(" + searchTextQuery + ") "; } return result; } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && !selectionMapping.isEmpty()) { String sqlSearchText = "%" + searchText + "%"; ctx.addStringParameter("lowerSearchTextParam", sqlSearchText); @@ -844,17 +849,17 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String singleEntityQuery(QueryContext ctx, SingleEntityFilter filter) { + private String singleEntityQuery(SqlQueryContext ctx, SingleEntityFilter filter) { ctx.addUuidParameter("entity_filter_single_entity_id", filter.getSingleEntity().getId()); return "e.id=:entity_filter_single_entity_id"; } - private String entityListQuery(QueryContext ctx, EntityListFilter filter) { + private String entityListQuery(SqlQueryContext ctx, EntityListFilter filter) { ctx.addUuidListParameter("entity_filter_entity_ids", filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toList())); return "e.id in (:entity_filter_entity_ids)"; } - private String entityNameQuery(QueryContext ctx, EntityNameFilter filter) { + private String entityNameQuery(SqlQueryContext ctx, EntityNameFilter filter) { ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter()); String nameColumn = getNameColumn(filter.getEntityType()); if (filter.getEntityNameFilter().startsWith("%") || filter.getEntityNameFilter().endsWith("%")) { @@ -864,7 +869,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return String.format("e.%s ilike concat(:entity_filter_name_filter, '%%')", nameColumn); } - private String typeQuery(QueryContext ctx, EntityFilter filter) { + private String typeQuery(SqlQueryContext ctx, EntityFilter filter) { List types; String name; String nameColumn; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 64e1f84db9..f6ca1581d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -37,7 +37,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { private long logQueriesThreshold; @Override - public void logQuery(QueryContext ctx, String query, long duration) { + public void logQuery(SqlQueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { String sqlToUse = substituteParametersInSqlString(query, ctx); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java new file mode 100644 index 0000000000..e486d3b645 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; + +@Service +@Slf4j +@ConditionalOnMissingBean(value = EdqsApiService.class, ignored = DummyEdqsApiService.class) +public class DummyEdqsApiService implements EdqsApiService { + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isEnabled() { + return false; + } + + @Override + public void setEnabled(boolean enabled) { + log.warn("Got request to enable EDQS API, but it isn't supported", new RuntimeException("stacktrace")); + } + + @Override + public boolean isSupported() { + return false; + } + + @Override + public boolean isAutoEnable() { + return false; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java new file mode 100644 index 0000000000..514e07c323 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsService; + +@Service +@ConditionalOnMissingBean(value = EdqsService.class, ignored = DummyEdqsService.class) +public class DummyEdqsService implements EdqsService { + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {} + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) {} + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) {} + + @Override + public void processSystemMsg(ToCoreEdqsMsg request) {} + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 949941a3b6..9755201fe9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -103,7 +103,7 @@ public class EntityKeyMapping { public static final List labeledEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, ADDITIONAL_INFO); public static final List contactBasedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, EMAIL, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO); - public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); + public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); public static final Set commonEntityFieldsSet = new HashSet<>(commonEntityFields); public static final Set relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO, RELATED_PARENT_ID)); @@ -265,7 +265,7 @@ public class EntityKeyMapping { return alias; } - public Stream toQueries(QueryContext ctx, EntityFilterType filterType) { + public Stream toQueries(SqlQueryContext ctx, EntityFilterType filterType) { if (hasFilter()) { String keyAlias = (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD) && getEntityKeyColumn() != null) ? "e" : alias; return keyFilters.stream().map(keyFilter -> @@ -275,7 +275,7 @@ public class EntityKeyMapping { } } - public String toLatestJoin(QueryContext ctx, EntityFilter entityFilter, EntityType entityType) { + public String toLatestJoin(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType) { String entityTypeStr; if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) { entityTypeStr = "entities.entity_type"; @@ -303,9 +303,9 @@ public class EntityKeyMapping { if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { scope = AttributeScope.CLIENT_SCOPE.getId(); } else if (entityKey.getType().equals(EntityKeyType.SHARED_ATTRIBUTE)) { - scope = AttributeScope.SHARED_SCOPE.getId();; + scope = AttributeScope.SHARED_SCOPE.getId(); ; } else { - scope = AttributeScope.SERVER_SCOPE.getId();; + scope = AttributeScope.SERVER_SCOPE.getId(); ; } query = String.format("%s AND %s.attribute_type=%s %s", query, alias, scope, filterQuery); } else { @@ -318,7 +318,7 @@ public class EntityKeyMapping { } } - private boolean hasFilterValues(QueryContext ctx) { + private boolean hasFilterValues(SqlQueryContext ctx) { return Arrays.stream(ctx.getParameterNames()).anyMatch(parameterName -> { return !parameterName.equals(getKeyId()) && parameterName.startsWith(alias); }); @@ -333,14 +333,14 @@ public class EntityKeyMapping { Collectors.joining(", ")); } - public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { + public static String buildLatestJoins(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { return latestMappings.stream() .filter(mapping -> !countQuery || mapping.hasFilter()) .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)) .collect(Collectors.joining(" ")); } - public static String buildQuery(QueryContext ctx, List mappings, EntityFilterType filterType) { + public static String buildQuery(SqlQueryContext ctx, List mappings, EntityFilterType filterType) { return mappings.stream() .flatMap(mapping -> mapping.toQueries(ctx, filterType)) .filter(StringUtils::isNotEmpty) @@ -510,12 +510,12 @@ public class EntityKeyMapping { return getValueAlias() + "_so_num"; } - private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter, + private String buildKeyQuery(SqlQueryContext ctx, String alias, KeyFilter keyFilter, EntityFilterType filterType) { return this.buildPredicateQuery(ctx, alias, keyFilter.getKey(), keyFilter.getPredicate(), filterType); } - private String buildPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (predicate.getType().equals(FilterPredicateType.COMPLEX)) { return this.buildComplexPredicateQuery(ctx, alias, key, (ComplexFilterPredicate) predicate, filterType); @@ -524,7 +524,7 @@ public class EntityKeyMapping { } } - private String buildComplexPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildComplexPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, ComplexFilterPredicate predicate, EntityFilterType filterType) { String result = predicate.getPredicates().stream() .map(keyFilterPredicate -> this.buildPredicateQuery(ctx, alias, key, keyFilterPredicate, filterType)) @@ -536,7 +536,7 @@ public class EntityKeyMapping { return result; } - private String buildSimplePredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildSimplePredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (key.getType().equals(EntityKeyType.ENTITY_FIELD)) { String field = (getEntityKeyColumn() != null) ? alias + "." + getEntityKeyColumn() : alias; @@ -571,7 +571,7 @@ public class EntityKeyMapping { } } - private String buildStringPredicateQuery(QueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { + private String buildStringPredicateQuery(SqlQueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { String operationField = field; String paramName = getNextParameterName(field); String value = stringFilterPredicate.getValue().getValue(); @@ -624,7 +624,7 @@ public class EntityKeyMapping { return String.format("((%s is not null and %s)", field, stringOperationQuery); } - private String buildNumericPredicateQuery(QueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { + private String buildNumericPredicateQuery(SqlQueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { String paramName = getNextParameterName(field); ctx.addDoubleParameter(paramName, numericFilterPredicate.getValue().getValue()); String numericOperationQuery = ""; @@ -651,7 +651,7 @@ public class EntityKeyMapping { return String.format("(%s is not null and %s)", field, numericOperationQuery); } - private String buildBooleanPredicateQuery(QueryContext ctx, String field, + private String buildBooleanPredicateQuery(SqlQueryContext ctx, String field, BooleanFilterPredicate booleanFilterPredicate) { String paramName = getNextParameterName(field); ctx.addBooleanParameter(paramName, booleanFilterPredicate.getValue().getValue()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java index ea15421fb7..86daeea77d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java @@ -17,5 +17,5 @@ package org.thingsboard.server.dao.sql.query; public interface QueryLogComponent { - void logQuery(QueryContext ctx, String query, long duration); + void logQuery(SqlQueryContext ctx, String query, long duration); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java index 0e33c1b44f..625458342c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java @@ -21,6 +21,7 @@ import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; import java.sql.Types; import java.util.HashMap; @@ -29,14 +30,14 @@ import java.util.Map; import java.util.UUID; @Slf4j -public class QueryContext implements SqlParameterSource { +public class SqlQueryContext implements SqlParameterSource { private static final UUIDJdbcType UUID_TYPE = UUIDJdbcType.INSTANCE; - private final QuerySecurityContext securityCtx; + private final QueryContext securityCtx; private final StringBuilder query; private final Map params; - public QueryContext(QuerySecurityContext securityCtx) { + public SqlQueryContext(QueryContext securityCtx) { this.securityCtx = securityCtx; query = new StringBuilder(); params = new HashMap<>(); @@ -48,7 +49,7 @@ public class QueryContext implements SqlParameterSource { if (oldParam != null && oldParam.value != null && !oldParam.value.equals(newParam.value)) { throw new RuntimeException("Parameter with name: " + name + " was already registered!"); } - if(value == null){ + if (value == null) { log.warn("[{}][{}][{}] Trying to set null value", getTenantId(), getCustomerId(), name); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java index dcfd008367..566e4eaddc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.QueueEntity; import org.thingsboard.server.dao.queue.QueueDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +39,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaQueueDao extends JpaAbstractDao implements QueueDao { +public class JpaQueueDao extends JpaAbstractDao implements QueueDao, TenantEntityDao { @Autowired private QueueRepository queueRepository; @@ -87,6 +88,11 @@ public class JpaQueueDao extends JpaAbstractDao implements Q .findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findQueuesByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index 15428dfa63..e09d01503c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -17,9 +17,11 @@ package org.thingsboard.server.dao.sql.queue; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -60,7 +62,7 @@ public class JpaQueueStatsDao extends JpaAbstractDao findByTenantId(TenantId tenantId, PageLink pageLink) { + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { return DaoUtil.toPageData(queueStatsRepository.findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } @@ -74,6 +76,11 @@ public class JpaQueueStatsDao extends JpaAbstractDao findNextBatch(UUID id, int batchSize) { + return queueStatsRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE_STATS; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java index bff8f05658..38e6fa9977 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.queue; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,6 +23,7 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.dao.model.sql.QueueStatsEntity; import java.util.List; @@ -45,4 +47,8 @@ public interface QueueStatsRepository extends JpaRepository findByTenantIdAndIdIn(UUID tenantId, List queueStatsIds); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.QueueStatsFields(q.id, q.createdTime," + + "q.tenantId, q.queueName, q.serviceId) FROM QueueStatsEntity q WHERE q.id > :id ORDER BY q.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 97e82b2021..7417418f54 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -127,6 +127,7 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { return service.submit(() -> checkRelation(tenantId, from, to, relationType, typeGroup)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index cebd95f08e..b4e8a21372 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.relation; +import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -84,4 +85,15 @@ public interface RelationRepository @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType and r.relationTypeGroup in :relationTypeGroups") void deleteByFromIdAndFromTypeAndRelationTypeGroupIn(@Param("fromId") UUID fromId, @Param("fromType") String fromType, @Param("relationTypeGroups") List relationTypeGroups); + @Query(value = "SELECT from_id, from_type, relation_type_group, relation_type, to_id, to_type, additional_info, version FROM relation" + + " WHERE (from_id, from_type, relation_type_group, relation_type, to_id, to_type) > " + + "(:fromId, :fromType, :relationTypeGroup, :relationType, :toId, :toType) ORDER BY " + + "from_id, from_type, relation_type_group, relation_type, to_id, to_type LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("toId") UUID toId, + @Param("toType") String toType, + @Param("batchSize") int batchSize); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java index 15517b4e3f..6cce9d76c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.TbResourceEntity; import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +39,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao { +public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao, TenantEntityDao { private final TbResourceRepository resourceRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java index ef7d28bb63..73e251b6db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.RpcEntity; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -39,7 +40,7 @@ import java.util.UUID; @Component @AllArgsConstructor @SqlDao -public class JpaRpcDao extends JpaAbstractDao implements RpcDao { +public class JpaRpcDao extends JpaAbstractDao implements RpcDao, TenantEntityDao { private final RpcRepository rpcRepository; @@ -74,6 +75,11 @@ public class JpaRpcDao extends JpaAbstractDao implements RpcDao return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findAllRpcByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.RPC; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 3d025fc469..77044d41dc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -17,9 +17,11 @@ package org.thingsboard.server.dao.sql.rule; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -33,6 +35,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -133,6 +136,16 @@ public class JpaRuleChainDao extends JpaAbstractDao return findRootRuleChainByTenantIdAndType(tenantId, RuleChainType.CORE); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return ruleChainRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index c3a8b447c5..8ecce0672f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import org.thingsboard.server.dao.rule.RuleNodeDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -40,7 +41,7 @@ import java.util.stream.Collectors; @Slf4j @Component @SqlDao -public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao { +public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao, TenantEntityDao { @Autowired private RuleNodeRepository ruleNodeRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 01bec2a846..cfa06caf14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.rule; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.RuleChainEntity; @@ -70,4 +72,7 @@ public interface RuleChainRepository extends JpaRepository :id ORDER BY r.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index bd9b20e01e..c27602421d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.settings; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; @@ -33,4 +35,6 @@ public interface AdminSettingsRepository extends JpaRepository findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 94f39d223c..68ce5e9d22 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -21,7 +21,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -32,7 +36,7 @@ import java.util.UUID; @Component @SqlDao @Slf4j -public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao { +public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao, TenantEntityDao { @Autowired private AdminSettingsRepository adminSettingsRepository; @@ -68,4 +72,9 @@ public class JpaAdminSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(adminSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index 0ba805f6e1..d031dbb8ee 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -16,11 +16,13 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; @@ -94,4 +96,9 @@ public class JpaTenantDao extends JpaAbstractDao implement .map(TenantId::fromUUID) .collect(Collectors.toList()); } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return tenantRepository.findNextBatch(id, Limit.of(batchSize)); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java index acc5feef31..839d62c48d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java @@ -16,11 +16,13 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -87,6 +89,11 @@ public class JpaTenantProfileDao extends JpaAbstractDao findNextBatch(UUID id, int batchSize) { + return tenantProfileRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.TENANT_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java index dc918c900e..c5759c8a0f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.dao.sql.tenant; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.dao.model.sql.TenantProfileEntity; import java.util.List; @@ -55,4 +57,8 @@ public interface TenantProfileRepository extends JpaRepository findByIdIn(List ids); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantProfileFields(t.id, t.createdTime, t.name," + + "t.isDefault) FROM TenantProfileEntity t WHERE t.id > :id ORDER BY t.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 8adb4e0261..bafa9a6fe6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.tenant; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; @@ -53,4 +55,8 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantFields(t.id, t.createdTime, t.title, t.version," + + "t.additionalInfo, t.country, t.state, t.city, t.address, t.address2, t.zip, t.phone, t.email, t.region) FROM TenantEntity t WHERE t.id > :id ORDER BY t.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index 0b27632f7a..98e62fc110 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.dao.sql.usagerecord; +import org.springframework.data.domain.Limit; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; +import java.util.List; import java.util.UUID; /** @@ -35,6 +40,8 @@ public interface ApiUsageStateRepository extends JpaRepository findAllByTenantId(UUID tenantId, Pageable pageable); + @Transactional @Modifying @Query("DELETE FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId") @@ -44,4 +51,10 @@ public interface ApiUsageStateRepository extends JpaRepository :id ORDER BY a.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java index 6c0ac91507..ec68fa34c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -15,18 +15,23 @@ */ package org.thingsboard.server.dao.sql.usagerecord; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; /** @@ -72,6 +77,16 @@ public class JpaApiUsageStateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(apiUsageStateRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return apiUsageStateRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.API_USAGE_STATE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java index d68dd395bd..4f9ff5d222 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.UserAuthSettings; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserAuthSettingsDao; @@ -31,7 +32,7 @@ import java.util.UUID; @Component @RequiredArgsConstructor @SqlDao -public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao { +public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao, TenantEntityDao { private final UserAuthSettingsRepository repository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java index 30b643afa5..bcae1dcdf4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java @@ -20,8 +20,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserCredentialsDao; @@ -34,7 +37,7 @@ import java.util.UUID; */ @Component @SqlDao -public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao { +public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao, TenantEntityDao { @Autowired private UserCredentialsRepository userCredentialsRepository; @@ -84,4 +87,9 @@ public class JpaUserCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index c0e5b530ef..35d15bab51 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; @@ -139,6 +141,16 @@ public class JpaUserDao extends JpaAbstractDao implements User return userRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return userRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.USER; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index bfe0e60556..646b11975e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -20,12 +20,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.common.data.settings.UserSettingsType; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserSettingsEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.user.UserSettingsDao; import org.thingsboard.server.dao.util.SqlDao; @@ -34,7 +36,7 @@ import java.util.List; @Slf4j @Component @SqlDao -public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService implements UserSettingsDao { +public class JpaUserSettingsDao implements UserSettingsDao, TenantEntityDao { @Autowired private UserSettingsRepository userSettingsRepository; @@ -66,4 +68,9 @@ public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService i return DaoUtil.convertDataList(userSettingsRepository.findByTypeAndPathExisting(type.name(), path)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java index 7ecb45d5e1..849cb22496 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -35,4 +37,7 @@ public interface UserAuthSettingsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java index 0dd3462c8b..51bc8702bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; @@ -52,4 +55,7 @@ public interface UserCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 731eda205e..0a30a859c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -15,15 +15,18 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.sql.UserEntity; import java.util.Collection; +import java.util.List; import java.util.UUID; /** @@ -71,4 +74,8 @@ public interface UserRepository extends JpaRepository { Long countByTenantId(UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.UserFields(u.id, u.createdTime, u.tenantId," + + "u.customerId, u.version, u.firstName, u.lastName, u.email, u.phone, u.additionalInfo) " + + "FROM UserEntity u WHERE u.id > :id ORDER BY u.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java index 12d43ff5ac..9423baafc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -36,4 +38,7 @@ public interface UserSettingsRepository extends JpaRepository findByTypeAndPathExisting(@Param("type") String type, @Param("path") String[] path); + @Query("SELECT s FROM UserSettingsEntity s WHERE s.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index df10804d50..c728f5d006 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -16,9 +16,11 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.page.PageData; @@ -30,6 +32,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeFilter; import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.common.data.widget.WidgetsBundleWidget; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.WidgetTypeDetailsEntity; import org.thingsboard.server.dao.model.sql.WidgetTypeInfoEntity; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; @@ -53,7 +56,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Component @SqlDao -public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao { +public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao, TenantEntityDao { @Autowired private WidgetTypeRepository widgetTypeRepository; @@ -256,10 +259,14 @@ public class JpaWidgetTypeDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); } + @Override + public List findNextBatch(UUID id, int batchSize) { + return widgetTypeRepository.findNextBatch(id, Limit.of(batchSize)); + } @Override public List findByTenantIdAndResourceLink(TenantId tenantId, String link, int limit) { @@ -271,4 +278,9 @@ public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetsBundleDao { +public class JpaWidgetsBundleDao extends JpaAbstractDao implements WidgetsBundleDao, TenantEntityDao { @Autowired private WidgetsBundleRepository widgetsBundleRepository; @@ -155,7 +158,17 @@ public class JpaWidgetsBundleDao extends JpaAbstractDao findByImageLink(String imageUrl, int limit) { - return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + } + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return widgetsBundleRepository.findNextBatch(id, Limit.of(batchSize)); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java index e148c10d74..ffa88b4d54 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.widget; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetTypeDetailsEntity; import org.thingsboard.server.dao.model.sql.WidgetTypeEntity; @@ -78,4 +80,7 @@ public interface WidgetTypeRepository extends JpaRepository findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields(w.id, w.createdTime, w.tenantId," + + "w.name, w.version) FROM WidgetTypeEntity w WHERE w.id > :id ORDER BY w.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index 516d6a6642..de778588dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -15,11 +15,14 @@ */ package org.thingsboard.server.dao.sql.widget; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; +import org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; @@ -139,4 +142,8 @@ public interface WidgetsBundleRepository extends JpaRepository findByImageUrl(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields(w.id, w.createdTime, w.tenantId," + + "w.alias, w.version) FROM WidgetsBundleEntity w WHERE w.id > :id ORDER BY w.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 664da621a5..6913aa8ea6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -@SuppressWarnings("UnstableApiUsage") @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { @@ -119,4 +118,5 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index be5d74f758..6e43034a44 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -33,14 +33,18 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.timeseries.TsLatestCacheKey; import org.thingsboard.server.dao.util.SqlTsLatestAnyDaoCachedRedis; import java.util.List; +import java.util.Map; import java.util.Optional; @Slf4j @@ -167,4 +171,5 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 5bb1c15e33..e8ef37b3b5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -24,6 +24,7 @@ import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,8 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; @@ -185,6 +188,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); } + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); return Futures.transformAsync(future, entryList -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java index 93ad8e8beb..53a824f026 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -22,6 +22,9 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -92,4 +95,9 @@ public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); } + @Override + public PageData findAll(PageLink pageLink) { + return DaoUtil.pageToPageData(keyDictionaryRepository.findAll(DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java index 13d7481b00..d264cd9966 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.dao.sqlts.dictionary; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -25,5 +28,7 @@ public interface KeyDictionaryRepository extends JpaRepository findByKeyId(int keyId); + @Query("SELECT e FROM KeyDictionaryEntry e ORDER BY e.keyId ASC") + Page findAll(Pageable pageable); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 77db6cd734..29d4377485 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -41,4 +41,10 @@ public interface TsKvLatestRepository extends JpaRepository findAllKeysByEntityIds(@Param("entityIds") List entityIds); + @Query(value = "SELECT entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version FROM ts_kv_latest WHERE (entity_id, key) > " + + "(:entityId, :key) ORDER BY entity_id, key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("key") int key, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java index e736bc791c..2a8269b15d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java @@ -37,10 +37,10 @@ public interface TenantDao extends Dao { * @return saved tenant object */ Tenant save(TenantId tenantId, Tenant tenant); - + /** * Find tenants by page link. - * + * * @param pageLink the page link * @return the list of tenant objects */ @@ -51,4 +51,5 @@ public interface TenantDao extends Dao { PageData findTenantsIds(PageLink pageLink); List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index f68bdbef38..19ce8e8993 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -26,6 +26,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; @@ -39,6 +41,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.Validator; @@ -55,7 +58,6 @@ import static org.thingsboard.server.common.data.StringUtils.isBlank; /** * @author Andrew Shvayka */ -@SuppressWarnings("UnstableApiUsage") @Service @Slf4j public class BaseTimeseriesService implements TimeseriesService { @@ -90,6 +92,9 @@ public class BaseTimeseriesService implements TimeseriesService { @Autowired private EntityViewService entityViewService; + @Autowired + private EdqsService edqsService; + @Override public ListenableFuture> findAllByQueries(TenantId tenantId, EntityId entityId, List queries) { validate(entityId); @@ -189,7 +194,10 @@ public class BaseTimeseriesService implements TimeseriesService { tsFutures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); } if (saveLatest) { - latestFutures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + latestFutures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), version -> { + edqsService.onUpdate(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); + return version; + }, MoreExecutors.directExecutor())); } } ListenableFuture dpsFuture = saveTs ? Futures.transform(Futures.allAsList(tsFutures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()) : Futures.immediateFuture(0); @@ -237,7 +245,7 @@ public class BaseTimeseriesService implements TimeseriesService { List> futures = new ArrayList<>(keys.size()); for (String key : keys) { DeleteTsKvQuery query = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false); - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } return Futures.allAsList(futures); } @@ -258,10 +266,20 @@ public class BaseTimeseriesService implements TimeseriesService { private void deleteAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, DeleteTsKvQuery query) { futures.add(Futures.transform(timeseriesDao.remove(tenantId, entityId, query), v -> null, MoreExecutors.directExecutor())); if (query.getDeleteLatest()) { - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } } + private ListenableFuture doRemove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return Futures.transform(timeseriesLatestDao.removeLatest(tenantId, entityId, query), result -> { + if (result.isRemoved()) { + Long version = result.getVersion(); + edqsService.onDelete(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, query.getKey(), version)); + } + return result; + }, MoreExecutors.directExecutor()); + } + private static void validate(EntityId entityId) { Validator.validateEntityId(entityId, id -> "Incorrect entityId " + id); } @@ -291,4 +309,5 @@ public class BaseTimeseriesService implements TimeseriesService { throw new IncorrectParameterException("Incorrect DeleteTsKvQuery. Key can't be empty"); } } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index d4b31f4b92..54a7e68725 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -36,13 +36,17 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -99,6 +103,7 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } + @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 7f7fe88936..32479301ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -22,8 +22,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import java.util.List; +import java.util.Map; import java.util.Optional; public interface TimeseriesLatestDao { @@ -49,4 +53,5 @@ public interface TimeseriesLatestDao { List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java index 29fe557822..ffff210693 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java @@ -19,10 +19,11 @@ import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface ApiUsageStateDao extends Dao { +public interface ApiUsageStateDao extends Dao, TenantEntityDao { /** * Save or update usage record object @@ -50,4 +51,5 @@ public interface ApiUsageStateDao extends Dao { void deleteApiUsageStateByTenantId(TenantId tenantId); void deleteApiUsageStateByEntityId(EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 3a867587e1..1282493650 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -113,6 +113,14 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A ApiUsageState saved = apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + eventPublisher.publishEvent(SaveEntityEvent.builder() + .tenantId(saved.getTenantId()) + .entityId(saved.getId()) + .entity(saved) + .created(true) + .broadcastEvent(false) + .build()); + List apiUsageStates = new ArrayList<>(); apiUsageStates.add(new BasicTsKvEntry(saved.getCreatedTime(), new StringDataEntry(ApiFeature.TRANSPORT.getApiStateKey(), ApiUsageStateValue.ENABLED.name()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index c1d2a3f2e1..b60b263ac8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 5bd34c4f1d..b425550e7e 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -14,22 +14,6 @@ -- limitations under the License. -- --- --- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, @@ -936,9 +920,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( configuration varchar(1000000), version BIGINT DEFAULT 1, debug_settings varchar(1024), - external_id UUID, - CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), - CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) + CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name) ); CREATE TABLE IF NOT EXISTS calculated_field_link ( diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index fbad72f371..25339244fd 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -77,7 +77,6 @@ import java.util.UUID; import static org.junit.Assert.assertNotNull; - @RunWith(SpringRunner.class) @ContextConfiguration(classes = AbstractServiceTest.class, loader = AnnotationConfigContextLoader.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @@ -131,7 +130,7 @@ public abstract class AbstractServiceTest { } public JsonNode readFromResource(String resourceName) throws IOException { - try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)){ + try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)) { return JacksonUtil.fromBytes(Objects.requireNonNull(is).readAllBytes()); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 5a783355d7..2985aa7620 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -101,20 +101,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { .hasMessage("Calculated Field with such name is already in exists!"); } - @Test - public void testSaveCalculatedFieldWithExistingExternalId() { - Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); - calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); - - calculatedFieldService.save(calculatedField); - - calculatedField.setName("Test 2"); - assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) - .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field with such external id already exists!"); - } - @Test public void testFindCalculatedFieldById() { CalculatedField savedCalculatedField = saveValidCalculatedField(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java index e69c52056a..7e3ec8ba56 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -29,6 +29,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit4.SpringRunner; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.QueryContext; import java.util.List; import java.util.UUID; @@ -48,7 +50,7 @@ import static org.mockito.Mockito.times; public class DefaultQueryLogComponentTest { private TenantId tenantId; - private QueryContext ctx; + private SqlQueryContext ctx; @SpyBean private DefaultQueryLogComponent queryLog; @@ -56,7 +58,7 @@ public class DefaultQueryLogComponentTest { @Before public void setUp() { tenantId = new TenantId(UUID.fromString("97275c1c-9cf2-4d25-a68d-933031158f84")); - ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); } @Test diff --git a/docker/.env b/docker/.env index 37c9768296..71722247df 100644 --- a/docker/.env +++ b/docker/.env @@ -14,6 +14,8 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor +EDQS_DOCKER_NAME=tb-edqs +EDQS_ENABLED=false TB_VERSION=latest diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index d5e60bee7f..5767026b11 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -128,6 +128,18 @@ function additionalStartupServices() { echo $ADDITIONAL_STARTUP_SERVICES } +function additionalComposeEdqsArgs() { + source .env + + if [ "$EDQS_ENABLED" = true ] + then + ADDITIONAL_COMPOSE_EDQS_ARGS="-f docker-compose.edqs.yml" + echo ADDITIONAL_COMPOSE_EDQS_ARGS + else + echo "" + fi +} + function permissionList() { PERMISSION_LIST=" 799 799 tb-node/log @@ -149,6 +161,12 @@ function permissionList() { " fi + if [ "$EDQS_ENABLED" = true ]; then + PERMISSION_LIST="$PERMISSION_LIST + 799 799 edqs/log + " + fi + CACHE="${CACHE:-redis}" case $CACHE in redis) diff --git a/docker/docker-compose.edqs.volumes.yml b/docker/docker-compose.edqs.volumes.yml new file mode 100644 index 0000000000..9d2ce946c8 --- /dev/null +++ b/docker/docker-compose.edqs.volumes.yml @@ -0,0 +1,30 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + tb-edqs-1: + volumes: + - tb-edqs-log-volume:/var/log/edqs + tb-edqs-2: + volumes: + - tb-edqs-log-volume:/var/log/edqs + +volumes: + tb-edqs-log-volume: + external: + name: ${TB_EDQS_LOG_VOLUME} diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml new file mode 100644 index 0000000000..67e9c987e3 --- /dev/null +++ b/docker/docker-compose.edqs.yml @@ -0,0 +1,57 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + tb-core1: + env_file: + - tb-core-edqs.env + tb-core2: + env_file: + - tb-core-edqs.env + tb-rule-engine1: + env_file: + - tb-rule-engine-edqs.env + tb-rule-engine2: + env_file: + - tb-rule-engine-edqs.env + tb-edqs-1: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + volumes: + - ./edqs/conf:/usr/share/edqs/conf + - ./edqs/log:/var/log/edqs + ports: + - "8080" + depends_on: + - zookeeper + - kafka + tb-edqs-2: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + volumes: + - ./edqs/conf:/usr/share/edqs/conf + - ./edqs/log:/var/log/edqs + ports: + - "8080" + depends_on: + - zookeeper + - kafka diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 1956e50eac..da09684d4f 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -49,6 +49,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? checkFolders --create || exit $? @@ -56,7 +58,8 @@ checkFolders --create || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d ${ADDITIONAL_STARTUP_SERVICES}" case $COMPOSE_VERSION in @@ -73,7 +76,8 @@ if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then fi COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \ tb-core1" diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 6b36f4be08..3124119abc 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ down -v" case $COMPOSE_VERSION in diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 3cdf10d00f..8b380f199b 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -29,10 +29,12 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + checkFolders --create || exit $? COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d" case $COMPOSE_VERSION in diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 670c44ca92..54386d10dd 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS}\ stop" case $COMPOSE_VERSION in diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 7a77241e48..de1fe0a89a 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -27,12 +27,16 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS_PULL="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ pull" COMPOSE_ARGS_BUILD="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d --no-deps --build" case $COMPOSE_VERSION in diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 3be5fbc14b..eca5d34957 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -42,21 +42,26 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? checkFolders --create || exit $? COMPOSE_ARGS_PULL="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ pull \ tb-core1" COMPOSE_ARGS_UP="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d ${ADDITIONAL_STARTUP_SERVICES}" COMPOSE_ARGS_RUN="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \ tb-core1" diff --git a/docker/edqs.env b/docker/edqs.env new file mode 100644 index 0000000000..2c07d6e80d --- /dev/null +++ b/docker/edqs.env @@ -0,0 +1,7 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 +TB_KAFKA_SERVERS=kafka:9092 +HTTP_BIND_PORT=8080 + +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus diff --git a/docker/edqs/conf/edqs.conf b/docker/edqs/conf/edqs.conf new file mode 100644 index 0000000000..8c6b5d1826 --- /dev/null +++ b/docker/edqs/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/edqs/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=tb-edqs.out +export LOADER_PATH=/usr/share/edqs/conf diff --git a/docker/edqs/conf/logback.xml b/docker/edqs/conf/logback.xml new file mode 100644 index 0000000000..40481a8c35 --- /dev/null +++ b/docker/edqs/conf/logback.xml @@ -0,0 +1,52 @@ + + + + + + + /var/log/edqs/${TB_SERVICE_ID}/tb-edqs.log + + /var/log/edqs/tb-edqs.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json new file mode 100644 index 0000000000..3e913d4856 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json @@ -0,0 +1,161 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "iteration": 1737564772936, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (objectType) (edqs_object_count{tenantId=~\"$tenantId\"})", + "interval": "", + "legendFormat": "{{objectType}}", + "refId": "A" + } + ], + "title": "EDQS object count", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(edqs_object_count, tenantId)", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": true, + "name": "tenantId", + "options": [], + "query": { + "query": "label_values(edqs_object_count, tenantId)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "EDQS", + "uid": "mK5A_DdHk", + "version": 9, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/tb-core-edqs.env b/docker/tb-core-edqs.env new file mode 100644 index 0000000000..45472ec907 --- /dev/null +++ b/docker/tb-core-edqs.env @@ -0,0 +1,5 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_MODE=remote +TB_EDQS_SYNC_ENABLED=true +TB_EDQS_API_SUPPORTED=true diff --git a/docker/tb-rule-engine-edqs.env b/docker/tb-rule-engine-edqs.env new file mode 100644 index 0000000000..82395ddcfe --- /dev/null +++ b/docker/tb-rule-engine-edqs.env @@ -0,0 +1,3 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_SYNC_ENABLED=true diff --git a/edqs/pom.xml b/edqs/pom.xml new file mode 100644 index 0000000000..07b25102d4 --- /dev/null +++ b/edqs/pom.xml @@ -0,0 +1,213 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + thingsboard + + edqs + jar + + ThingsBoard Entity Data Query Service Application + https://thingsboard.io + + + UTF-8 + ${basedir}/.. + java + false + process-resources + package + edqs + ${project.build.directory}/windows + true + ThingsBoard Entity Data Query Service + org.thingsboard.server.edqs.ThingsboardEdqsApplication + + + + + org.thingsboard.common + edqs + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.apache.curator + curator-recipes + + + com.google.protobuf + protobuf-java + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + + com.sun.winsw + winsw + bin + exe + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.assertj + assertj-core + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + true + + edqs.yml + + + + ${project.basedir}/src/main/resources + false + + edqs.yml + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + thingsboard + + + **/nosql/*Test.java + + + **/*Test.java + **/*TestSuite.java + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-winsw-service + package + + + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + org.codehaus.mojo + build-helper-maven-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/edqs/src/main/conf/edqs.conf b/edqs/src/main/conf/edqs.conf new file mode 100644 index 0000000000..3f96fd590d --- /dev/null +++ b/edqs/src/main/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/edqs/src/main/conf/logback.xml b/edqs/src/main/conf/logback.xml new file mode 100644 index 0000000000..850a28b212 --- /dev/null +++ b/edqs/src/main/conf/logback.xml @@ -0,0 +1,49 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java new file mode 100644 index 0000000000..1f1152af68 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class DummyQueueRoutingInfoService implements QueueRoutingInfoService { + + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java new file mode 100644 index 0000000000..4e16e5e16a --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class DummyTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + return null; + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java b/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java new file mode 100644 index 0000000000..4d0858ad5a --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.edqs.state.EdqsStateService; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/edqs") +public class EdqsController { + + private final EdqsStateService edqsStateService; + + @GetMapping("/ready") + public ResponseEntity isReady() { + if (edqsStateService.isReady()) { + return ResponseEntity.ok().build(); + } else { + return ResponseEntity.badRequest().build(); + } + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java new file mode 100644 index 0000000000..00b3fdbd26 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.edqs", "org.thingsboard.server.queue.edqs", "org.thingsboard.server.queue.discovery", "org.thingsboard.server.queue.kafka", + "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment", "org.thingsboard.server.common.stats"}) +@Slf4j +public class ThingsboardEdqsApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "edqs"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardEdqsApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } + +} diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml new file mode 100644 index 0000000000..1d7f111e9a --- /dev/null +++ b/edqs/src/main/resources/edqs.yml @@ -0,0 +1,201 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +server: + # Server bind-address + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port + port: "${HTTP_BIND_PORT:8080}" + +# Application info parameters +app: + # Application version + version: "@project.version@" + +# Zookeeper connection parameters +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. + # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. + # The delay is recommended because the initialization of rule chain actors is time-consuming. Avoiding unnecessary recalculations during a restart can enhance system performance and stability. + recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" + +spring: + main: + allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. + +# Queue configuration parameters +queue: + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) + prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). + edqs: + # Number of partitions for EDQS topics + partitions: "${TB_EDQS_PARTITIONS:12}" + # EDQS partitioning strategy: tenant (partitions are resolved and distributed by tenant id) or none (partitions are resolved by message key; each instance has all the partitions) + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS requests topic + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + # EDQS responses topic + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + # Poll interval for EDQS topics + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + # Maximum amount of pending requests to EDQS + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout for requests to EDQS + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + stats: + # Enable/disable statistics for EDQS + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" + + kafka: + # Kafka Bootstrap nodes in "host:port" format + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + ssl: + # Enable/Disable SSL Kafka communication + enabled: "${TB_KAFKA_SSL_ENABLED:false}" + # The location of the trust store file + truststore.location: "${TB_KAFKA_SSL_TRUSTSTORE_LOCATION:}" + # The password of trust store file if specified + truststore.password: "${TB_KAFKA_SSL_TRUSTSTORE_PASSWORD:}" + # The location of the key store file. This is optional for the client and can be used for two-way authentication for the client + keystore.location: "${TB_KAFKA_SSL_KEYSTORE_LOCATION:}" + # The store password for the key store file. This is optional for the client and only needed if ‘ssl.keystore.location’ is configured. Key store password is not supported for PEM format + keystore.password: "${TB_KAFKA_SSL_KEYSTORE_PASSWORD:}" + # The password of the private key in the key store file or the PEM key specified in ‘keystore.key’ + key.password: "${TB_KAFKA_SSL_KEY_PASSWORD:}" + # The number of acknowledgments the producer requires the leader to have received before considering a request complete. This controls the durability of records that are sent. The following settings are allowed:0, 1 and all + acks: "${TB_KAFKA_ACKS:all}" + # Number of retries. Resend any record whose send fails with a potentially transient error + retries: "${TB_KAFKA_RETRIES:1}" + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # Default batch size. This setting gives the upper bound of the batch size to be sent + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + # The maximum size of a request in bytes. This setting will limit the number of record batches the producer will send in a single request to avoid sending huge requests + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + # The maximum number of unacknowledged requests the client will send on a single connection before blocking + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + # The total bytes of memory the producer can use to buffer records waiting to be sent to the server + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + # The multiple copies of data over the multiple brokers of Kafka + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + # The maximum delay between invocations of poll() method when using consumer group management. This places an upper bound on the amount of time that the consumer can be idle before fetching more records + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + # The maximum number of records returned in a single call of poll() method + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + # The maximum amount of data per-partition the server will return. Records are fetched in batches by the consumer + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + # The maximum amount of data the server will return. Records are fetched in batches by the consumer + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none + # Enable/Disable using of Confluent Cloud + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + # The endpoint identification algorithm used by clients to validate server hostname. The default value is https + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + # The mechanism used to authenticate Schema Registry requests. SASL/PLAIN should only be used with TLS/SSL as a transport layer to ensure that clear passwords are not transmitted on the wire without encryption + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + # Using JAAS Configuration for specifying multiple SASL mechanisms on a broker + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + edqs.events: + # Key-value properties for Kafka consumer for edqs.events topic + - key: max.poll.records + # Max poll records for edqs.events topic + value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}" + edqs.state: + # Key-value properties for Kafka consumer for edqs.state topic + - key: max.poll.records + # Max poll records for edqs.state topic + value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" + + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + consumer-stats: + # Prints lag between consumer group offset and last messages offset in Kafka topics + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + # Statistics printing interval for Kafka's consumer-groups stats + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + # Time to wait for the stats-loading requests to Kafka to finish + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + +# General service parameters +service: + type: "${TB_SERVICE_TYPE:edqs}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + edqs: + # EDQS instances with the same label will share the same list of partitions + label: "${TB_EDQS_LABEL:}" + +# Metrics parameters +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + system_info: + # Persist frequency of system info (CPU, memory usage, etc.) in seconds + persist_frequency: "${METRICS_SYSTEM_INFO_PERSIST_FREQUENCY_SECONDS:60}" + # TTL in days for system info timeseries + ttl: "${METRICS_SYSTEM_INFO_TTL_DAYS:7}" + +# General management parameters +management: + endpoints: + web: + exposure: + # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). + include: '${METRICS_ENDPOINTS_EXPOSE:info}' + health: + elasticsearch: + # Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck + enabled: "false" diff --git a/edqs/src/main/resources/logback.xml b/edqs/src/main/resources/logback.xml new file mode 100644 index 0000000000..5a6e9e3e24 --- /dev/null +++ b/edqs/src/main/resources/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java new file mode 100644 index 0000000000..01a7495148 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -0,0 +1,252 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.EdqsConverter; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@RunWith(SpringRunner.class) +@Configuration +@ComponentScan({"org.thingsboard.server.edqs.repo", "org.thingsboard.server.edqs.util"}) +@EntityScan("org.thingsboard.server.edqs") +@TestPropertySource(locations = {"classpath:edqs-test.properties"}) +@TestExecutionListeners({ + DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class}) +public abstract class AbstractEDQTest { + + @Autowired + protected DefaultEdqsRepository repository; + @Autowired + protected EdqsConverter edqsConverter; + + protected final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + protected final CustomerId customerId = new CustomerId(UUID.randomUUID()); + + protected final UUID defaultAssetProfileId = UUID.randomUUID(); + protected final UUID defaultDeviceProfileId = UUID.randomUUID(); + + @Before + public final void before() { + AssetProfile ap = new AssetProfile(new AssetProfileId(defaultAssetProfileId)); + ap.setName("default"); + ap.setDefault(true); + addOrUpdate(EntityType.ASSET_PROFILE, ap); + + DeviceProfile dp = new DeviceProfile(new DeviceProfileId(defaultDeviceProfileId)); + dp.setName("default"); + dp.setDefault(true); + dp.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, dp); + + createCustomer(customerId.getId(), null, "Customer A"); + } + + @After + public final void after() { + repository.clear(); + } + + protected void createCustomer(UUID id, UUID parentCustomerId, String title) { + Customer entity = new Customer(); + entity.setId(new CustomerId(id)); + entity.setTitle(title); + addOrUpdate(EntityType.CUSTOMER, entity); + } + + + protected UUID createDevice(String name) { + return createDevice(null, defaultDeviceProfileId, name); + } + + protected UUID createDevice(CustomerId customerId, String name) { + return createDevice(customerId.getId(), defaultDeviceProfileId, name); + } + + protected UUID createDevice(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Device entity = new Device(); + entity.setId(new DeviceId(entityId)); + if (profileId != null) { + entity.setDeviceProfileId(new DeviceProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createDashboard(String name) { + UUID entityId = UUID.randomUUID(); + Dashboard entity = new Dashboard(); + entity.setId(new DashboardId(entityId)); + entity.setTitle(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createView(String name) { + return createView(null, "default", name); + } + + protected UUID createView(CustomerId customerId, String name) { + return createView(customerId.getId(), "default", name); + } + + protected UUID createView(UUID customerId, String type, String name) { + UUID entityId = UUID.randomUUID(); + EntityView entity = new EntityView(); + entity.setId(new EntityViewId(entityId)); + entity.setType(type); + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ENTITY_VIEW, entity); + return entityId; + } + + protected UUID createEdge(String name) { + return createEdge(null, "default", name); + } + + protected UUID createEdge(CustomerId customerId, String name) { + return createEdge(customerId.getId(), "default", name); + } + + protected UUID createEdge(UUID customerId, String type, String name) { + UUID id = UUID.randomUUID(); + Edge edge = new Edge(); + edge.setId(new EdgeId(id)); + edge.setTenantId(tenantId); + if (customerId != null) { + edge.setCustomerId(new CustomerId(customerId)); + } + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + addOrUpdate(EntityType.EDGE, edge); + return id; + } + + + protected UUID createAsset(String name) { + return createAsset(null, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, String name) { + return createAsset(customerId, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Asset entity = new Asset(); + entity.setId(new AssetId(entityId)); + if (profileId != null) { + entity.setAssetProfileId(new AssetProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ASSET, entity); + return entityId; + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, String type) { + createRelation(fromType, fromId, toType, toId, RelationTypeGroup.COMMON, type); + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, RelationTypeGroup group, String type) { + addOrUpdate(new EntityRelation(EntityIdFactory.getByTypeAndUuid(fromType, fromId), EntityIdFactory.getByTypeAndUuid(toType, toId), type, group)); + } + + + protected boolean checkContains(PageData data, UUID entityId) { + return data.getData().stream().anyMatch(r -> r.getEntityId().getId().equals(entityId)); + } + + protected List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value) { + KeyFilter filter = new KeyFilter(); + filter.setKey(new EntityKey(keyType, key)); + filter.setValueType(EntityKeyValueType.STRING); + StringFilterPredicate predicate = new StringFilterPredicate(); + predicate.setValue(FilterPredicateValue.fromString(value)); + predicate.setOperation(operation); + predicate.setIgnoreCase(true); + filter.setPredicate(predicate); + return Collections.singletonList(filter); + } + + protected void addOrUpdate(EntityType entityType, Object entity) { + addOrUpdate(EdqsConverter.toEntity(entityType, entity)); + } + + protected void addOrUpdate(EdqsObject edqsObject) { + byte[] serialized = edqsConverter.serialize(edqsObject.type(), edqsObject); + edqsObject = edqsConverter.deserialize(edqsObject.type(), serialized); + repository.get(tenantId).addOrUpdate(edqsObject); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java new file mode 100644 index 0000000000..e9470ca0b3 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.UUID; + +public class ApiUsageStateFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + Tenant entity = new Tenant(); + entity.setId(tenantId); + entity.setTitle("test tenant"); + addOrUpdate(EntityType.TENANT, entity); + } + + @After + public void tearDown() { + } + + @Test + public void testFindCustomerApiUsageState() { + UUID customerId = UUID.randomUUID(); + createCustomer(customerId, null, "Customer A"); + + ApiUsageState apiUsageState = buildApiUsageState(customerId); + addOrUpdate(EntityType.API_USAGE_STATE, apiUsageState); + + var result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(new CustomerId(customerId)), false); + + Assert.assertEquals(1, result.getTotalElements()); + var customer = result.getData().get(0); + Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + private ApiUsageState buildApiUsageState(UUID customerId) { + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setId(new ApiUsageStateId(UUID.randomUUID())); + apiUsageState.setTenantId(tenantId); + apiUsageState.setEntityId(new CustomerId(customerId)); + apiUsageState.setTransportState(ApiUsageStateValue.ENABLED); + apiUsageState.setReExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setJsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setTbelExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setDbStorageState(ApiUsageStateValue.ENABLED); + apiUsageState.setSmsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setEmailExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setAlarmExecState(ApiUsageStateValue.ENABLED); + return apiUsageState; + } + + private static EntityDataQuery getEntityDataQuery(CustomerId customerId) { + ApiUsageStateFilter filter = new ApiUsageStateFilter(); + filter.setCustomerId(customerId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("Customer A")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java new file mode 100644 index 0000000000..1f90babf01 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class AssetSearchQueryFilterTest extends AbstractEDQTest { + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(null, assetProfileId.getId(), "root"); + UUID asset1 = createAsset(null, assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(null, assetProfileId.getId(), "A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset + PageData relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets with max level = 1 + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + + // find all assets with asset type = default + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all assets last level only, level = 2 + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets last level only, level = 1 + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindCustomerAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(customerId.getId(), assetProfileId.getId(), "root"); + UUID asset1 = createAsset(customerId.getId(), assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(customerId.getId(), assetProfileId.getId(), "A2"); + UUID asset3 = createAsset(customerId.getId(), defaultAssetProfileId, "A3"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset3, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset with profile "Office" + PageData relationsResult = findData(customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office", "default")); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + Assert.assertTrue(checkContains(relationsResult, asset3)); + + // find all assets with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List assetTypes) { + AssetSearchQueryFilter filter = new AssetSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setAssetTypes(assetTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "A"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java new file mode 100644 index 0000000000..0961a7c12c --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java @@ -0,0 +1,185 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AssetTypeFilterTest extends AbstractEDQTest { + + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + private final AssetProfileId assetProfileId2 = new AssetProfileId(UUID.randomUUID()); + private Asset asset; + private Asset asset2; + private Asset asset3; + + @Before + public void setUp() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + AssetProfile assetProfile2 = new AssetProfile(assetProfileId2); + assetProfile2.setName("Street"); + assetProfile2.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile2); + + asset = buildAsset(assetProfileId, "Office 1"); + asset2 = buildAsset(assetProfileId, "Office 2"); + asset3 = buildAsset(assetProfileId2, "Abbey Road"); + + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(EntityType.ASSET, asset2); + addOrUpdate(EntityType.ASSET, asset3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantAsset() { + // find asset with type "Office" + var result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(Collections.singletonList("Office"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Office 1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(asset.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Office" and "Street" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office", "Street"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var third = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Abbey Road")).findAny(); + assertThat(third).isPresent(); + assertThat(third.get().getEntityId()).isEqualTo(asset3.getId()); + assertThat(third.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Supermarket" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with name "%Office%" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), "%Office%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with name "Office 1" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), "Office 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find asset with name "%Super%" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), "%Super%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with key filter: name contains "Office" + KeyFilter containsNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "office", true); + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with key filter: name starts with "office" and matches case + KeyFilter startsWithNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "office", false); + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerAsset() { + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(new LatestTsKv(asset.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getAssetTypeQuery(List.of("Office"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + asset.setCustomerId(customerId); + addOrUpdate(EntityType.ASSET, asset); + + result = repository.findEntityDataByQuery(tenantId, customerId, getAssetTypeQuery(List.of("Office"), null, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(asset.getId(), first.getEntityId()); + Assert.assertEquals("Office 1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Asset buildAsset(AssetProfileId assetProfileId, String assetName) { + Asset asset = new Asset(); + asset.setId(new AssetId(UUID.randomUUID())); + asset.setTenantId(tenantId); + asset.setAssetProfileId(assetProfileId); + asset.setName(assetName); + asset.setCreatedTime(42L); + return asset; + } + + private static EntityDataQuery getAssetTypeQuery(List assetTypes, String assetNameRegex, List keyFilters) { + AssetTypeFilter filter = new AssetTypeFilter(); + filter.setAssetTypes(assetTypes); + filter.setAssetNameFilter(assetNameRegex); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getAssetNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java new file mode 100644 index 0000000000..b3f15c2f61 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java @@ -0,0 +1,150 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class DeviceSearchQueryFilterTest extends AbstractEDQTest { + private final DeviceProfileId deviceProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset("A1"); + UUID asset2 = createAsset("A2"); + UUID device1 = createDevice(null, deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(null, deviceProfileId.getId(), "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of asset A1 + PageData relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all devices with max level = 1 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices with asset type = default + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all devices last level only, level = 2 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device2)); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices last level only, level = 1 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindCustomerDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of type "thermostat" + PageData relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat", "default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all assets with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List deviceTypes) { + DeviceSearchQueryFilter filter = new DeviceSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setDeviceTypes(deviceTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "D"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java new file mode 100644 index 0000000000..e07dfc98e4 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java @@ -0,0 +1,141 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.UUID; + +public class DeviceTypeFilterTest extends AbstractEDQTest { + + private final DeviceProfileId loraProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + DeviceProfile deviceProfile = new DeviceProfile(loraProfileId); + deviceProfile.setName("LoRa"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setDeviceProfileId(loraProfileId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getDeviceTypeQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("default"), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(loraProfileId); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceTypeQuery(String deviceType) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(Collections.singletonList(deviceType)); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java new file mode 100644 index 0000000000..f0911570ea --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class EdgeSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindDevicesManagesByTenant() { + UUID edge1 = createEdge("E1"); + UUID edge2 = createEdge("E2"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID device3 = createDevice("D3"); + + createRelation(EntityType.EDGE, edge1, EntityType.DEVICE, device1, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device2, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device3, "Manages"); + + // find devices managed by edge + PageData relationsResult = findData(null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + + // find devices managed by edge with non-existing type + relationsResult = findData(null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 1, false, Arrays.asList("non-existing type")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, true, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + } + + @Test + public void testFindCustomerEdges() { + UUID edge1 = createEdge(customerId, "E1"); + UUID edge2 = createEdge(customerId, "E2"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge2, "Manages"); + + // find all edges managed by customer + PageData relationsResult = findData(customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + Assert.assertTrue(checkContains(relationsResult, edge2)); + + // find all edges managed by customer with non-existing type + relationsResult = findData(customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("non existing")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List edgeTypes) { + EdgeSearchQueryFilter filter = new EdgeSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEdgeTypes(edgeTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "E"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java new file mode 100644 index 0000000000..1d7bbef011 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EdgeTypeFilterTest extends AbstractEDQTest { + + private Edge edge; + private Edge edge2; + private Edge edge3; + + + @Before + public void setUp() { + edge = buildEdge("default", "Edge 1"); + edge2 = buildEdge("default", "Edge 2"); + edge3 = buildEdge("edge v2", "Edge 3"); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEdges() { + // find edges with type "default" + var result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(Collections.singletonList("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find edges with types "default" and "edge v2" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(Arrays.asList("default", "edge v2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 3")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(edge3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Edge%" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), "%Edge%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Edge 1" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), "Edge 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Edge 4%" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), "%Edge 4%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Edge" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Edge", true); + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), null, List.of(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "edge" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "edge", false); + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), null, List.of(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEdges() { + addOrUpdate(new LatestTsKv(edge.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getEdgeTypeQuery(List.of("default"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + edge.setCustomerId(customerId); + edge2.setCustomerId(customerId); + edge3.setCustomerId(customerId); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEdgeTypeQuery(List.of("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Edge buildEdge(String type, String name) { + Edge edge = new Edge(); + edge.setId(new EdgeId(UUID.randomUUID())); + edge.setTenantId(tenantId); + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + return edge; + } + + private static EntityDataQuery getEdgeTypeQuery(List edgeTypes, String edgeNameFilter, List keyFilters) { + EdgeTypeFilter filter = new EdgeTypeFilter(); + filter.setEdgeTypes(edgeTypes); + filter.setEdgeNameFilter(edgeNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java new file mode 100644 index 0000000000..dc48bd5438 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class EntityListFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityListDataQuery(EntityType.DEVICE,List.of(device.getId().getId().toString(), device2.getId().getId().toString())), false); + Assert.assertEquals(2, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityListDataQuery(EntityType entityType, List ids) { + EntityListFilter filter = new EntityListFilter(); + filter.setEntityType(entityType); + filter.setEntityList(ids); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "LoRa-"); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java new file mode 100644 index 0000000000..3e47c245ef --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java @@ -0,0 +1,131 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.UUID; + +public class EntityNameFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("%1"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("L%"), false); + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceNameQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceNameQuery(String entityNameFilter) { + EntityNameFilter filter = new EntityNameFilter(); + filter.setEntityType(EntityType.DEVICE); + filter.setEntityNameFilter(entityNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java new file mode 100644 index 0000000000..8fab142dc7 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntityTypeFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDeviceEntities() { + // find all tenant devices + var result = repository.findEntityDataByQuery(tenantId, null, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("LoRa-1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(device.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(device.getCreatedTime())); + assertThat(first.get().getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()).isEqualTo("enabled"); + + // find all tenant devices with filter by name + KeyFilter keyFilter = getDeviceNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, getEntityTypeQuery(EntityType.DEVICE, List.of(keyFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset entities + result = repository.findEntityDataByQuery(tenantId, null, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDeviceEntities() { + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityTypeQuery(EntityType.DEVICE, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityTypeQuery(EntityType entityType, List keyFilters) { + EntityTypeFilter filter = new EntityTypeFilter(); + filter.setEntityType(entityType); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getDeviceNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java new file mode 100644 index 0000000000..fb32759045 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java @@ -0,0 +1,130 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class EntityViewSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantEntityViews() { + UUID asset1 = createAsset("A1"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID deviceView1 = createView("V1"); + UUID deviceView2 = createView("V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of asset A1 + PageData relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views with max level = 1 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with type "day 1" + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("day 1")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + } + + @Test + public void testFindCustomerDevices() { + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + UUID deviceView1 = createView(customerId.getId(), "day 1", "V1"); + UUID deviceView2 = createView(customerId.getId(), "day 1", "V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of type "day 1" + PageData relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views of type "day 2" + relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 2")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List entityViewTypes) { + EntityViewSearchQueryFilter filter = new EntityViewSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEntityViewTypes(entityViewTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "V"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java new file mode 100644 index 0000000000..bf1d329127 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EntityViewTypeFilterTest extends AbstractEDQTest { + + private EntityView entityView; + private EntityView entityView2; + private EntityView entityView3; + + + @Before + public void setUp() { + entityView = buildEntityView("day 1", "day 1 lora 1 view"); + entityView2 = buildEntityView("day 1", "day 1 lora 2 view"); + entityView3 = buildEntityView("day 2", "day 2 lora 1 view"); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityView() { + // find entity view with type "day 1" + var result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with types "day 1" and "day 2" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Arrays.asList("day 1", "day 2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 2 lora 1 view")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(entityView3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Lora%" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 1 lora%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Lora 1 device view" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), "day 1 lora 1 view", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Parking sensor%" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 3 lora%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Lora" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "lora" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "lora", false); + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityView() { + addOrUpdate(new LatestTsKv(entityView.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + entityView.setCustomerId(customerId); + entityView2.setCustomerId(customerId); + entityView3.setCustomerId(customerId); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private EntityView buildEntityView(String type, String name) { + EntityView entityView = new EntityView(); + entityView.setId(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setType(type); + entityView.setName(name); + entityView.setCreatedTime(42L); + return entityView; + } + + private static EntityDataQuery getEntityViewTypeQuery(List assetTypes, String entityViewNameFilter, List keyFilters) { + EntityViewTypeFilter filter = new EntityViewTypeFilter(); + filter.setEntityViewTypes(assetTypes); + filter.setEntityViewNameFilter(entityViewNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java new file mode 100644 index 0000000000..094d46f977 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java @@ -0,0 +1,160 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class RelationsQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice("NOT MATCHING D3"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da3, "Contains"); + + PageData relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta2)); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da2)); + } + + @Test + public void testFindTenantDevicesLastLevelOnly() { + UUID root = createAsset("T ROOT"); + + UUID ta1 = createAsset("T A1 NO MORE RELATIONS"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice(customerId, "T D3"); + UUID da4 = createDevice(customerId, "T D4"); // Lvl 4 + + // ROOT --Contains--> A1, A2; A2 --Contains--> D1, D2; D2 --Contains--> D3. + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da2, "Contains"); + createRelation(EntityType.ASSET, da2, EntityType.DEVICE, da3, "Contains"); + createRelation(EntityType.ASSET, da3, EntityType.DEVICE, da4, "Contains"); + + PageData relationsResult = filter(null, new AssetId(root), 1, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, ta2)); + + relationsResult = filter(null, new AssetId(root), 2, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da2)); + + relationsResult = filter(null, new AssetId(root), 3, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da3)); + + relationsResult = filter(null, new AssetId(root), 4, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da4)); + + } + + @Test + public void testFindCustomerDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice(customerId, "T D1"); + UUID da2 = createDevice("T D2"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + + PageData relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData filter(EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(null, rootId, relationEntityTypeFilters); + } + + private PageData filter(CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(customerId, rootId, 3, false, relationEntityTypeFilters); + } + + private PageData filter(CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setRootEntity(rootId); + filter.setFilters(Arrays.asList(relationEntityTypeFilters)); + filter.setDirection(EntitySearchDirection.FROM); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "T"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java new file mode 100644 index 0000000000..6c7444c92a --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -0,0 +1,434 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate.BooleanOperation; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate.NumericOperation; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class RepositoryUtilsTest { + + private static Stream deviceNameFilters() { + return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet device 123", getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "ra"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "device"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet 123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet "), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet 123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameFilters") + public void testFilterByDeviceName(String deviceName, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream createdTimeFilters() { + return Stream.of(Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 999), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 999), false) + ); + } + + @ParameterizedTest + @MethodSource("createdTimeFilters") + public void testFilterDevicesByCreatedTime(long createdTime, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().createdTime(createdTime).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceNameAndTypeFilter() { + return Stream.of( + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "thermostat")), true), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "thermostat")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false)); + } + + @ParameterizedTest + @MethodSource("deviceNameAndTypeFilter") + public void testFilterByDeviceNameAndDeviceType(String deviceName, String deviceType, List keyFilters, boolean result) { + UUID deviceProfileId = UUID.randomUUID(); + EntityProfileData deviceProfile = new EntityProfileData(deviceProfileId, EntityType.DEVICE_PROFILE); + deviceProfile.setFields(DeviceProfileFields.builder().name(deviceType).build()); + + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).deviceProfileId(deviceProfileId).type(deviceType).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceNameComplexFilters() { + return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "net")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the")), false), + Arguments.of("loranet123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.OR, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameComplexFilters") + public void testFilterByDeviceNameComplexFilters(String deviceName, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceTemperatureFilters() { + return Stream.of(Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.9), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.8), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.9), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 22.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 23.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureFilters") + public void testFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureComplexFilters() { + return Stream.of(Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), true), + Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER, 23.5, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), false), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.OR, NumericOperation.GREATER, 35), true), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.AND, NumericOperation.EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureComplexFilters") + public void testComplexFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceHumidityFilters() { + return Stream.of(Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 59), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 59), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 60), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceHumidityFilters") + public void testFilterByDeviceHumidity(long humidityValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureAndHumidityFilters() { + return Stream.of(Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.EQUAL, 60)), true), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER, 23), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.9), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureAndHumidityFilters") + public void testFilterByDeviceTemperatureAndHumidity(double tempValue, long humidityValue, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceVersionAttributeFilters() { + return Stream.of(Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, true), true), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, false), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, true), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, false), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceVersionAttributeFilters") + public void testFilterByDeviceVersionAttribute(Boolean active, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceActiveAndVersionFilters() { + return Stream.of(Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), true), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceActiveAndVersionFilters") + public void testFilterByActiveAndVersionAttributes(Boolean active, String version, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(1, AttributeScope.CLIENT_SCOPE, new StringDataPoint(System.currentTimeMillis(), version)); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static EdqsFilter getVersionAttributeFilter(StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.CLIENT_ATTRIBUTE, "version", 1); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + + private static EdqsFilter getActiveAttributeFilter(BooleanOperation operation, boolean predicateValue) { + BooleanFilterPredicate filterPredicate = new BooleanFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromBoolean(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.SERVER_ATTRIBUTE, "active", 2); + return new EdqsFilter(key, EntityKeyValueType.BOOLEAN, filterPredicate); + } + + private static EdqsFilter getTemperatureFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("temperature", 5, operation, predicateValue); + } + + private static EdqsFilter getHumidityFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("humidity", 6, operation, predicateValue); + } + + private static EdqsFilter getTimeseriesFilter(String key, Integer keysId, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey newKey = new DataKey(EntityKeyType.TIME_SERIES, key, keysId); + return new EdqsFilter(newKey, EntityKeyValueType.NUMERIC, filterPredicate); + } + + private static EdqsFilter getNameFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("name", operation, predicateValue); + } + + private static EdqsFilter getTypeFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("type", operation, predicateValue); + } + + private static EdqsFilter getStringEntityFieldFilter(String fieldName, StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + private static EdqsFilter getCreatedTimeFilter(NumericOperation operation, double predicateValue) { + return getDatetimeEntityFieldFilter("createdTime", operation, predicateValue); + } + + private static EdqsFilter getDatetimeEntityFieldFilter(String fieldName, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.DATE_TIME, filterPredicate); + } + + private static EdqsFilter getComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2, + ComplexOperation complexOperation2, NumericOperation operation3, double predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexNumericFilterPredicate(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + NumericFilterPredicate filterPredicate2 = new NumericFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromDouble(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2, + ComplexOperation complexOperation2, StringOperation operation3, String predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation3); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue3)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexStringFilterPredicate(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + StringFilterPredicate filterPredicate2 = new StringFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromString(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java new file mode 100644 index 0000000000..133816f576 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.UUID; + +public class SingleEntityFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(1, result.getTotalElements()); + first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { + SingleEntityFilter filter = new SingleEntityFilter(); + filter.setSingleEntity(deviceId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/resources/edqs-test.properties b/edqs/src/test/resources/edqs-test.properties new file mode 100644 index 0000000000..8a041c7407 --- /dev/null +++ b/edqs/src/test/resources/edqs-test.properties @@ -0,0 +1,2 @@ +zk.enabled=false +service.type=edqs diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 3c067ee641..65def6a964 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -51,6 +51,7 @@ public class ContainerTestSuite { private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; + private static final String TB_EDQS_LOG_REGEXP = ".*All partitions processed.*"; private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); @@ -114,6 +115,8 @@ public class ContainerTestSuite { List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.edqs.yml"), + new File(targetDir + "docker-compose.edqs.volumes.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.mosquitto.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), @@ -174,6 +177,8 @@ public class ContainerTestSuite { .withExposedService("broker", 1883) .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-rule-engine1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-rule-engine2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) @@ -182,7 +187,9 @@ public class ContainerTestSuite { .waitingFor("tb-lwm2m-transport", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-1", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-2", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 94d0c64fa0..d74438a428 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; @@ -56,6 +57,9 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rpc.Rpc; @@ -66,7 +70,6 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import static io.restassured.RestAssured.given; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; @@ -110,6 +113,20 @@ public class TestRestClient { requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); } + public void resetToken() { + token = null; + refreshToken = null; + } + + public Tenant postTenant(Tenant tenant) { + return given().spec(requestSpec).body(tenant) + .post("/api/tenant") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Tenant.class); + } + public Device postDevice(String accessToken, Device device) { return given().spec(requestSpec).body(device) .pathParams("accessToken", accessToken) @@ -220,6 +237,15 @@ public class TestRestClient { .as(JsonNode.class); } + public JsonNode getLatestTelemetry(EntityId entityId) { + return given().spec(requestSpec) + .get("/api/plugins/telemetry/" + entityId.getEntityType().name() + "/" + entityId.getId() + "/values/timeseries") + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + public JsonPath postProvisionRequest(String provisionRequest) { return given().spec(requestSpec) .body(provisionRequest) @@ -479,6 +505,28 @@ public class TestRestClient { .as(User.class); } + public UserId createUserAndLogin(User user, String password) { + UserId userId = postUser(user).getId(); + getAndSetUserToken(userId); + return userId; + } + + public void getAndSetUserToken(UserId id) { + ObjectNode tokenInfo = given().spec(requestSpec) + .get("/api/user/" + id.getId().toString() + "/token") + .then() + .extract() + .as(ObjectNode.class); + token = tokenInfo.get("token").asText(); + refreshToken = tokenInfo.get("refreshToken").asText(); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + protected void resetTokens() { + this.token = null; + this.refreshToken = null; + } + public void deleteUser(UserId userId) { given().spec(requestSpec) .delete("/api/user/{userId}", userId.getId()) @@ -643,4 +691,45 @@ public class TestRestClient { } return urlParams; } + + public PageData postEntityDataQuery(EntityDataQuery entityDataQuery) { + return given().spec(requestSpec).body(entityDataQuery) + .post("/api/entitiesQuery/find") + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef<>() {}); + } + + public Long postCountDataQuery(EntityCountQuery entityCountQuery) { + return given().spec(requestSpec).body(entityCountQuery) + .post("/api/entitiesQuery/count") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Long.class); + } + + public Boolean isEdqsApiEnabled() { + return given().spec(requestSpec) + .get("/api/edqs/enabled") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Boolean.class); + } + + public void assignDeviceToCustomer(CustomerId customerId, DeviceId id) { + given().spec(requestSpec) + .post("/api/customer/" + customerId.getId().toString() + "/device/" + id.getId().toString()) + .then() + .statusCode(HTTP_OK); + } + + public void deleteTenant(TenantId tenantId) { + given().spec(requestSpec) + .delete("/api/tenant/" + tenantId.getId().toString()) + .then() + .statusCode(HTTP_OK); + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index a71abe1781..9c1a90a4f1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -48,6 +48,7 @@ public class ThingsBoardDbInstaller { private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; + private final static String TB_EDQS_LOG_VOLUME = "tb-edqs-log-test-volume"; private final static String JAVA_OPTS = "-Xmx512m"; private final DockerComposeExecutor dockerCompose; @@ -65,6 +66,7 @@ public class ThingsBoardDbInstaller { private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; private final String tbVcExecutorLogVolume; + private final String tbEdqsLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -103,6 +105,7 @@ public class ThingsBoardDbInstaller { tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; + tbEdqsLogVolume = project + "_" + TB_EDQS_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -119,6 +122,7 @@ public class ThingsBoardDbInstaller { env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + env.put("TB_EDQS_LOG_VOLUME", tbEdqsLogVolume); if (IS_REDIS_CLUSTER) { for (int i = 0; i < 6; i++) { env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); @@ -189,6 +193,9 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbEdqsLogVolume); + dockerCompose.invokeDocker(); + StringBuilder additionalServices = new StringBuilder(); if (IS_HYBRID_MODE) { additionalServices.append(" cassandra"); @@ -220,7 +227,8 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true " + + "tb-core1"); dockerCompose.invokeCompose(); } finally { @@ -240,6 +248,7 @@ public class ThingsBoardDbInstaller { copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); + copyLogs(tbEdqsLogVolume, "./target/tb-edqs-logs/"); StringJoiner rmVolumesCommand = new StringJoiner(" ") .add("volume rm -f") @@ -251,6 +260,7 @@ public class ThingsBoardDbInstaller { .add(tbMqttTransportLogVolume) .add(tbSnmpTransportLogVolume) .add(tbVcExecutorLogVolume) + .add(tbEdqsLogVolume) .add(resolveRedisComposeVolumeLog()); if (IS_HYBRID_MODE) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java new file mode 100644 index 0000000000..53d8a72e7f --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -0,0 +1,214 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.edqs; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.ui.utils.EntityPrototypes; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomer; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomerAdmin; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultDeviceProfile; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultTenantAdmin; + +@DisableUIListeners +public class EdqsEntityDataQueryTest extends AbstractContainerTest { + + private TenantId tenantId; + private CustomerId customerId; + private TenantId tenantId2; + private CustomerId customerId2; + private UserId tenantAdminId; + private UserId customerUserId; + private UserId tenant2AdminId; + private UserId customer2UserId; + private final List tenantDevices = new ArrayList<>(); + private final List tenant2Devices = new ArrayList<>(); + private final String deviceProfile = "LoRa-" + RandomStringUtils.randomAlphabetic(10); + + @BeforeClass + public void beforeClass() throws Exception { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + await().atMost(60, TimeUnit.SECONDS).until(() -> testRestClient.isEdqsApiEnabled()); + + tenantId = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenantAdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId, "tenantAdmin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenantDevices, 97); + customerId = testRestClient.postCustomer(defaultCustomer(tenantId, "Customer")).getId(); + customerUserId = testRestClient.postUser(defaultCustomerAdmin(tenantId, customerId, "customerUser@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId, tenantDevices, 12); + + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + tenantId2 = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenant2AdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId2, "tenant2Admin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenant2Devices, 97); + customerId2 = testRestClient.postCustomer(defaultCustomer(tenantId2, "Customer")).getId(); + customer2UserId = testRestClient.postUser(defaultCustomerAdmin(tenantId2, customerId2, "customer2User@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId2, tenant2Devices, 12); + } + + @BeforeMethod + public void beforeMethod() { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + } + + @AfterClass + public void afterClass() { + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.deleteTenant(tenantId); + testRestClient.deleteTenant(tenantId2); + } + + @Test + public void testSysAdminCountEntitiesByQuery() { + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery query = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).compareTo(97L * 2) >= 0); + + testRestClient.getAndSetUserToken(tenantAdminId); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getAndSetUserToken(tenant2AdminId); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + } + + @Test + public void testRetrieveTenantDevicesByDeviceTypeFilter() { + // login tenant admin + testRestClient.getAndSetUserToken(tenantAdminId); + checkUserDevices(tenantDevices); + + // login customer user + testRestClient.getAndSetUserToken(customerUserId); + checkUserDevices(tenantDevices.subList(0, 12)); + + // login other tenant admin + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getAndSetUserToken(tenant2AdminId); + checkUserDevices(tenant2Devices); + } + + private void checkUserDevices(List devices) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of(deviceProfile)); + filter.setDeviceNameFilter(""); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestFields = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestFields, null); + + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery countQuery = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(countQuery).intValue() == devices.size()); + + PageData result = testRestClient.postEntityDataQuery(query); + assertThat(result.getTotalElements()).isEqualTo(devices.size()); + List retrievedDevices = result.getData(); + + assertThat(retrievedDevices).hasSize(10); + List retrievedDeviceNames = retrievedDevices.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).toList(); + assertThat(retrievedDeviceNames).containsExactlyInAnyOrderElementsOf(devices.stream().map(Device::getName).toList().subList(0, 10)); + + //check temperature + for (int i = 0; i < 10; i++) { + Map> latest = retrievedDevices.get(i).getLatest(); + String name = latest.get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(latest.get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).isEqualTo(name.substring(name.length() - 1)); + } + } + + private String createDevices(String deviceType, List tenantDevices, int deviceCount) throws InterruptedException { + String prefix = StringUtils.randomAlphabetic(5); + for (int i = 0; i < deviceCount; i++) { + Device device = new Device(); + device.setName(prefix + "Device" + i); + device.setType(deviceType); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + //TO make sure devices have different created time + Thread.sleep(1); + String token = RandomStringUtils.randomAlphabetic(10); + Device saved = testRestClient.postDevice(token, device); + tenantDevices.add(saved); + + // save timeseries data + testRestClient.postTelemetry(token, createDeviceTelemetry(i)); + } + return deviceType; + } + + private void assignDevicesToCustomer(CustomerId customerId, List devices, int deviceCount) { + for (int i = 0; i < deviceCount; i++) { + Device device = devices.get(i); + testRestClient.assignDeviceToCustomer(customerId, device.getId()); + } + } + + protected ObjectNode createDeviceTelemetry(int temperature) { + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put("temperature", temperature); + return objectNode; + } + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java index ce5fee1f6f..91620271b7 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -37,12 +38,26 @@ import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfilePr import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.security.Authority; public class EntityPrototypes { + public static Tenant defaultTenantPrototype(String tenantName) { + Tenant tenant = new Tenant(); + tenant.setTitle(tenantName); + return tenant; + } + + public static Customer defaultCustomer(TenantId tenantId, String title) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(title); + return customer; + } + public static Customer defaultCustomerPrototype(String entityName) { Customer customer = new Customer(); customer.setTitle(entityName); @@ -169,6 +184,23 @@ public class EntityPrototypes { return user; } + public static User defaultTenantAdmin(TenantId tenantId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setEmail(email); + user.setAuthority(Authority.TENANT_ADMIN); + return user; + } + + public static User defaultCustomerAdmin(TenantId tenantId, CustomerId customerId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setCustomerId(customerId); + user.setEmail(email); + user.setAuthority(Authority.CUSTOMER_USER); + return user; + } + public static User defaultUser(String email, CustomerId customerId, String name) { User user = new User(); user.setEmail(email); diff --git a/msa/black-box-tests/src/test/resources/connectivity.xml b/msa/black-box-tests/src/test/resources/connectivity.xml index 2bde3f0a3f..425fbd67eb 100644 --- a/msa/black-box-tests/src/test/resources/connectivity.xml +++ b/msa/black-box-tests/src/test/resources/connectivity.xml @@ -22,6 +22,7 @@ + \ No newline at end of file diff --git a/msa/edqs/docker/Dockerfile b/msa/edqs/docker/Dockerfile new file mode 100644 index 0000000000..e9099c09c5 --- /dev/null +++ b/msa/edqs/docker/Dockerfile @@ -0,0 +1,31 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk17:bookworm-slim + +COPY start-tb-edqs.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-edqs.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chown -R ${pkg.user}:${pkg.user} /tmp && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-edqs.sh"] \ No newline at end of file diff --git a/msa/edqs/docker/start-tb-edqs.sh b/msa/edqs/docker/start-tb-edqs.sh new file mode 100755 index 0000000000..deb0f70eff --- /dev/null +++ b/msa/edqs/docker/start-tb-edqs.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER=${pkg.installFolder}/conf +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.edqs.ThingsboardEdqsApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=$CONF_FOLDER/logback.xml \ + org.springframework.boot.loader.launch.PropertiesLauncher diff --git a/msa/edqs/pom.xml b/msa/edqs/pom.xml new file mode 100644 index 0000000000..f22cb0187c --- /dev/null +++ b/msa/edqs/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + msa + + org.thingsboard.msa + edqs + pom + + ThingsBoard Entity Data Query Microservice + https://thingsboard.io + ThingsBoard Entity Data Query Microservice + + + UTF-8 + ${basedir}/../.. + edqs + tb-edqs + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard + edqs + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-edqs + package + + copy + + + + + org.thingsboard + edqs + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/pom.xml b/msa/pom.xml index 5ae0e903c1..98e820daaf 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -48,6 +48,7 @@ transport js-executor monitoring + edqs diff --git a/pom.xml b/pom.xml index 218a021cee..c0dc7384f7 100755 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ 1.7.0 4.4.0 2.2.14 + 0.6.12 3.12.1 2.0.0-M15 2.10.1 @@ -83,7 +84,7 @@ 3.9.2 3.25.5 1.63.0 - 1.2.5 + 1.2.6 1.18.32 1.2.5 1.2.5 @@ -165,7 +166,7 @@ 1.6.1 2.19.0 9.2.0 - + 1.1.10.5 9.10.0 @@ -174,6 +175,7 @@ common rule-engine dao + edqs transport ui-ngx tools @@ -1035,6 +1037,11 @@ coap-server ${project.version} + + org.thingsboard.common + edqs + ${project.version} + org.thingsboard.common.script script-api @@ -2283,6 +2290,11 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.xerial.snappy + snappy-java + ${snappy.version} + org.rocksdb rocksdbjni diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceStateManager.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/DeviceStateManager.java similarity index 88% rename from rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceStateManager.java rename to rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/DeviceStateManager.java index fb3e282c9a..887f7ecaa2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceStateManager.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/DeviceStateManager.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; -public interface RuleEngineDeviceStateManager { +public interface DeviceStateManager { void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback); @@ -29,4 +29,6 @@ public interface RuleEngineDeviceStateManager { void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback); + void onDeviceInactivityTimeoutUpdate(TenantId tenantId, DeviceId deviceId, long inactivityTimeout, TbCallback callback); + } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineCalculatedFieldQueueService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineCalculatedFieldQueueService.java new file mode 100644 index 0000000000..6ab40b79c2 --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineCalculatedFieldQueueService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import com.google.common.util.concurrent.FutureCallback; + +public interface RuleEngineCalculatedFieldQueueService { + + void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback callback); + + void pushRequestToQueue(AttributesSaveRequest request, FutureCallback callback); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 6ba4eae0e4..e473df4c54 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -280,7 +280,7 @@ public interface TbContext { DeviceCredentialsService getDeviceCredentialsService(); - RuleEngineDeviceStateManager getDeviceStateManager(); + DeviceStateManager getDeviceStateManager(); String getDeviceStateNodeRateLimitConfig(); @@ -360,6 +360,8 @@ public interface TbContext { CalculatedFieldService getCalculatedFieldService(); + RuleEngineCalculatedFieldQueueService getCalculatedFieldQueueService(); + boolean isExternalNodeForceAck(); /** diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java index 2ff431c928..c3f6b5c74c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java @@ -58,8 +58,7 @@ public class TimeseriesDeleteRequest implements CalculatedFieldSystemAwareReques private TbMsgType tbMsgType; private FutureCallback> callback; - Builder() { - } + Builder() {} public Builder tenantId(TenantId tenantId) { this.tenantId = tenantId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java index 3aed05d372..c9a50cf88d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java @@ -17,7 +17,7 @@ package org.thingsboard.rule.engine.action; import lombok.extern.slf4j.Slf4j; import org.springframework.util.ConcurrentReferenceHashMap; -import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; @@ -119,7 +119,7 @@ public class TbDeviceStateNode implements TbNode { TenantId tenantId = ctx.getTenantId(); long eventTs = msg.getMetaDataTs(); - RuleEngineDeviceStateManager deviceStateManager = ctx.getDeviceStateManager(); + DeviceStateManager deviceStateManager = ctx.getDeviceStateManager(); TbCallback callback = getMsgEnqueuedCallback(ctx, msg); switch (event) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java new file mode 100644 index 0000000000..b2cce52c87 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import com.google.gson.JsonParser; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "calculated fields", + configClazz = EmptyNodeConfiguration.class, + nodeDescription = "Pushes incoming messages to calculated fields service", + nodeDetails = "Node enables the processing of calculated fields without persisting incoming messages to the database. " + + "By default, the processing of calculated fields is triggered by the save attributes and save time series nodes. " + + "This rule node accepts the same messages as these nodes but allows you to trigger the processing of calculated " + + "fields independently, ensuring that derived data can be computed and utilized in real time without storing the original message in the database.", + configDirective = "tbNodeEmptyConfig", + icon = "published_with_changes" +) +public class TbCalculatedFieldsNode implements TbNode { + + private EmptyNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + switch (msg.getInternalType()) { + case POST_TELEMETRY_REQUEST -> processPostTelemetryRequest(ctx, msg); + case POST_ATTRIBUTES_REQUEST -> processPostAttributesRequest(ctx, msg); + default -> ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); + } + } + + private void processPostTelemetryRequest(TbContext ctx, TbMsg msg) { + Map> tsKvMap = JsonConverter.convertToTelemetry(JsonParser.parseString(msg.getData()), System.currentTimeMillis()); + + if (tsKvMap.isEmpty()) { + ctx.tellSuccess(msg); + return; + } + + List tsKvEntryList = new ArrayList<>(); + for (Map.Entry> tsKvEntry : tsKvMap.entrySet()) { + for (KvEntry kvEntry : tsKvEntry.getValue()) { + tsKvEntryList.add(new BasicTsKvEntry(tsKvEntry.getKey(), kvEntry)); + } + } + + TimeseriesSaveRequest timeseriesSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(ctx.getTenantId()) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entries(tsKvEntryList) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .callback(new TelemetryNodeCallback(ctx, msg)) + .build(); + + ctx.getCalculatedFieldQueueService().pushRequestToQueue(timeseriesSaveRequest, timeseriesSaveRequest.getCallback()); + } + + private void processPostAttributesRequest(TbContext ctx, TbMsg msg) { + List newAttributes = new ArrayList<>(JsonConverter.convertToAttributes(JsonParser.parseString(msg.getData()))); + + if (newAttributes.isEmpty()) { + ctx.tellSuccess(msg); + return; + } + + AttributesSaveRequest attributesSaveRequest = AttributesSaveRequest.builder() + .tenantId(ctx.getTenantId()) + .entityId(msg.getOriginator()) + .scope(AttributeScope.valueOf(msg.getMetaData().getValue(SCOPE))) + .entries(newAttributes) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .callback(new TelemetryNodeCallback(ctx, msg)) + .build(); + ctx.getCalculatedFieldQueueService().pushRequestToQueue(attributesSaveRequest, attributesSaveRequest.getCallback()); + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java index b31dffdc01..2c3c61150e 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java @@ -29,7 +29,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.test.util.ReflectionTestUtils; import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; @@ -66,7 +66,7 @@ public class TbDeviceStateNodeTest { @Mock private TbContext ctxMock; @Mock - private RuleEngineDeviceStateManager deviceStateManagerMock; + private DeviceStateManager deviceStateManagerMock; @Captor private ArgumentCaptor callbackCaptor; private TbDeviceStateNode node; @@ -263,7 +263,7 @@ public class TbDeviceStateNodeTest { @ParameterizedTest @MethodSource - public void givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback(TbMsgType supportedEventType, BiConsumer> actionVerification) { + public void givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback(TbMsgType supportedEventType, BiConsumer> actionVerification) { // GIVEN given(ctxMock.getTenantId()).willReturn(TENANT_ID); given(ctxMock.getDeviceStateNodeRateLimitConfig()).willReturn("1:1"); @@ -297,10 +297,10 @@ public class TbDeviceStateNodeTest { private static Stream givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback() { return Stream.of( - Arguments.of(TbMsgType.CONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceConnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.ACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceActivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.DISCONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceDisconnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.INACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceInactivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())) + Arguments.of(TbMsgType.CONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceConnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.ACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceActivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.DISCONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceDisconnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.INACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceInactivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())) ); } diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index fe5b0f7b52..66c0cb609e 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -23,6 +23,7 @@ import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/m import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityTestScriptResult } from '@shared/models/entity.models'; +import { CalculatedFieldEventBody } from '@shared/models/event.models'; @Injectable({ providedIn: 'root' @@ -53,4 +54,8 @@ export class CalculatedFieldsService { public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } + + public getLatestCalculatedFieldDebugEvent(id: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${id}/debug`, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index ddf52642f0..5689f176e9 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -14,7 +14,11 @@ /// limitations under the License. /// -import { EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; +import { + DateEntityTableColumn, + EntityTableColumn, + EntityTableConfig +} from '@home/models/entity/entities-table-config.models'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { TranslateService } from '@ngx-translate/core'; import { Direction } from '@shared/models/page/sort-order'; @@ -42,6 +46,7 @@ import { getCalculatedFieldArgumentsEditorCompleter, getCalculatedFieldArgumentsHighlights, CalculatedFieldTypeTranslations, + CalculatedFieldType, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -51,6 +56,7 @@ import { import { ImportExportService } from '@shared/import-export/import-export.service'; import { isObject } from '@core/utils'; import { EntityDebugSettingsService } from '@home/components/entity/debug/entity-debug-settings.service'; +import { DatePipe } from '@angular/common'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -67,6 +73,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig, private destroyRef: DestroyRef, @@ -104,11 +111,20 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('expression', 'calculated-fields.expression', '33%', entity => entity.configuration?.expression); + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '300px'); expressionColumn.sortable = false; + expressionColumn.cellContentFunction = entity => { + const expressionLabel = this.getExpressionLabel(entity); + return expressionLabel.length < 45 ? expressionLabel : `${expressionLabel.substring(0, 44)}…`; + } + expressionColumn.cellTooltipFunction = entity => { + const expressionLabel = this.getExpressionLabel(entity); + return expressionLabel.length < 45 ? null : expressionLabel + }; + this.columns.push(new DateEntityTableColumn('createdTime', 'common.created-time', this.datePipe, '150px')); this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); this.columns.push(new EntityTableColumn('type', 'common.type', '50px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); this.columns.push(expressionColumn); @@ -143,6 +159,14 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index c1dd0bbd5a..91a4d69252 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -34,6 +34,7 @@ import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { EntityDebugSettingsService } from '@home/components/entity/debug/entity-debug-settings.service'; +import { DatePipe } from '@angular/common'; @Component({ selector: 'tb-calculated-fields-table', @@ -56,6 +57,9 @@ export class CalculatedFieldsTableComponent { private translate: TranslateService, private dialog: MatDialog, private store: Store, + private datePipe: DatePipe, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, private cd: ChangeDetectorRef, private renderer: Renderer2, private importExportService: ImportExportService, @@ -68,6 +72,7 @@ export class CalculatedFieldsTableComponent { this.calculatedFieldsService, this.translate, this.dialog, + this.datePipe, this.entityId(), this.store, this.destroyRef, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 6bab01f976..162bd3aa1e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -20,18 +20,27 @@ - +
{{ 'common.name' | translate }}
 - -
{{ argument.argumentName }}
+ +
+
{{ argument.argumentName }}
+ +
- + {{ 'entity.entity-type' | translate }} - +
@if (argument.refEntityId?.entityType === ArgumentEntityType.Tenant) { {{ 'calculated-fields.argument-current-tenant' | translate }} @@ -98,7 +107,7 @@
- } +
{{ 'calculated-fields.output' | translate }}
@@ -154,26 +152,35 @@ }
@if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { - - - {{ (outputFormGroup.get('type').value === OutputType.Timeseries - ? 'calculated-fields.timeseries-key' - : 'calculated-fields.attribute-key') - | translate }} - - - @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { - - @if (outputFormGroup.get('name').hasError('required')) { - {{ 'common.hint.key-required' | translate }} - } @else if (outputFormGroup.get('name').hasError('pattern')) { - {{ 'common.hint.key-pattern' | translate }} - } @else if (outputFormGroup.get('name').hasError('maxlength')) { - {{ 'common.hint.key-max-length' | translate }} - } - - } - +
+ + + {{ (outputFormGroup.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate }} + + + @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { + + @if (outputFormGroup.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputFormGroup.get('decimalsByDefault').errors && outputFormGroup.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + +
} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss index 0e994ed825..3bb2759274 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -13,39 +13,37 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -:host { - .dialog-container { - width: 869px; - max-width: 100%; - } - .script-lang-chip { - line-height: 20px; - font-size: 14px; - font-weight: 500; - color: white; - border-radius: 100px; - width: 70px; - display: flex; - justify-content: center; - margin-top: 2px; - margin-right: 4px; - } +.calculated-field-dialog-container { + width: 869px; + max-width: 100%; +} + +.tbel-script-lang-chip { + line-height: 20px; + font-size: 14px; + font-weight: 500; + color: white; + border-radius: 100px; + width: 70px; + min-width: 70px; + display: flex; + justify-content: center; + margin-top: 2px; + margin-right: 4px; } -:host ::ng-deep { - .expression-edit { - .ace_tb { - &.ace_calculated-field { - &-key { - color: #C52F00; - } - &-ts, &-time-window, &-values, &-value, &-func { - color: #7214D0; - } - &-start-ts, &-end-ts, &-limit { - color: #185F2A; - } +.tb-js-func { + .ace_tb { + &.ace_calculated-field { + &-key { + color: #C52F00; + } + &-ts, &-time-window, &-values, &-value, &-func { + color: #7214D0; + } + &-start-ts, &-end-ts, &-limit { + color: #185F2A; } } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c3e3d52a72..0c3e7464cd 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, DestroyRef, Inject } from '@angular/core'; +import { Component, DestroyRef, Inject, ViewEncapsulation } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -24,6 +24,7 @@ import { DialogComponent } from '@shared/components/dialog.component'; import { CalculatedField, CalculatedFieldConfiguration, + calculatedFieldDefaultScript, CalculatedFieldDialogData, CalculatedFieldType, CalculatedFieldTypeTranslations, @@ -32,33 +33,36 @@ import { OutputType, OutputTypeTranslations } from '@shared/models/calculated-field.models'; -import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { digitsRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map, startWith } from 'rxjs/operators'; +import { map, startWith, switchMap } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { Observable } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', templateUrl: './calculated-field-dialog.component.html', styleUrls: ['./calculated-field-dialog.component.scss'], + encapsulation: ViewEncapsulation.None }) export class CalculatedFieldDialogComponent extends DialogComponent { fieldFormGroup = this.fb.group({ - name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + name: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], type: [CalculatedFieldType.SIMPLE], debugSettings: [], configuration: this.fb.group({ arguments: this.fb.control({}), - expressionSIMPLE: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], - expressionSCRIPT: [], + expressionSIMPLE: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], + expressionSCRIPT: [calculatedFieldDefaultScript], output: this.fb.group({ - name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + name: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], - type: [OutputType.Timeseries] + type: [OutputType.Timeseries], + decimalsByDefault: [null as number, [Validators.min(0), Validators.max(15), Validators.pattern(digitsRegex)]], }), }), }); @@ -118,9 +122,18 @@ export class CalculatedFieldDialogComponent extends DialogComponent { + const calculatedFieldId = this.data.value?.id?.id; + let testScriptDialogResult$: Observable; + + if (calculatedFieldId) { + testScriptDialogResult$ = this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) + .pipe( + switchMap(event => { + const args = event?.arguments ? JSON.parse(event.arguments) : null; + return this.data.getTestScriptDialogFn(this.fromGroupValue, args, false); + }), + takeUntilDestroyed(this.destroyRef) + ) + } else { + testScriptDialogResult$ = this.data.getTestScriptDialogFn(this.fromGroupValue, null, false); + } + + testScriptDialogResult$.subscribe(expression => { this.configFormGroup.get('expressionSCRIPT').setValue(expression); this.configFormGroup.get('expressionSCRIPT').markAsDirty(); }); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 9d9614ba28..7e41f60968 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -17,7 +17,7 @@ import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; -import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charsWithNumRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, ArgumentEntityTypeParamsMap, @@ -71,10 +71,10 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { }), refEntityKey: this.fb.group({ type: [ArgumentType.LatestTelemetry, [Validators.required]], - key: [''], + key: ['', [Validators.pattern(oneSpaceInsideRegex)]], scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }, [Validators.required]], }), - defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], + defaultValue: ['', [Validators.pattern(oneSpaceInsideRegex)]], limit: [{ value: this.defaultLimit, disabled: !this.maxDataPointsPerRollingArg }], timeWindow: [MINUTE * 15], }); @@ -142,6 +142,10 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { if (refEntityId.entityType === ArgumentEntityType.Tenant) { refEntityId.id = this.tenantId; } + if (value.defaultValue) { + value.defaultValue = value.defaultValue.trim(); + } + value.refEntityKey.key = value.refEntityKey.key.trim(); this.argumentsDataApplied.emit({ value, index: this.index }); } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss index 2187b47e0d..d944e2a683 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -71,20 +71,4 @@ background-image: url("../../../../../../../assets/split.js/grips/vertical.png"); } } - - .expression-edit { - .ace_tb { - &.ace_calculated-field { - &-key { - color: #C52F00; - } - &-ts, &-time-window, &-values, &-value, &-func { - color: #7214D0; - } - &-start-ts, &-end-ts, &-limit { - color: #185F2A; - } - } - } - } } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 6c334d2a5e..56a17cc7ae 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -369,7 +369,7 @@ export class EventTableConfig extends EntityTableConfig { (entity) => entity.body.entityId, { name: this.translate.instant('event.copy-entity-id'), - icon: 'content_paste', + icon: 'content_copy', style: { padding: '4px', 'font-size': '16px', @@ -389,7 +389,7 @@ export class EventTableConfig extends EntityTableConfig { false, { name: this.translate.instant('event.copy-message-id'), - icon: 'content_paste', + icon: 'content_copy', style: { padding: '4px', 'font-size': '16px', diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html index 053d0f5006..d80892664d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -84,6 +84,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index e0a88a3e09..d55515a959 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -25,7 +25,11 @@ import { TranslateService } from '@ngx-translate/core'; import { DialogService } from '@core/services/dialog.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; import { Observable, of } from 'rxjs'; -import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + typesWithCalculatedFields +} from '@shared/models/vc.models'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; @@ -43,6 +47,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit isReadOnly: Observable; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private adminService: AdminService, private dialogService: DialogService, @@ -104,7 +110,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit branch: null, saveAttributes: true, saveRelations: false, - saveCredentials: true + saveCredentials: true, + saveCalculatedFields: true, }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -206,7 +213,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit branch: [config.branch, []], saveRelations: [config.saveRelations, []], saveAttributes: [config.saveAttributes, []], - saveCredentials: [config.saveCredentials, []] + saveCredentials: [config.saveCredentials, []], + saveCalculatedFields: [config.saveCalculatedFields, []] }) } ); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index b9a9a6c01c..8fdf22f6cd 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -72,6 +72,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index 8b16905587..8618e405b0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -33,7 +33,8 @@ import { EntityTypeVersionCreateConfig, exportableEntityTypes, SyncStrategy, - syncStrategyTranslationMap + syncStrategyTranslationMap, + typesWithCalculatedFields } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -79,6 +80,8 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements loading = true; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private translate: TranslateService, private fb: UntypedFormBuilder, @@ -150,6 +153,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements saveRelations: [config.saveRelations, []], saveAttributes: [config.saveAttributes, []], saveCredentials: [config.saveCredentials, []], + saveCalculatedFields: [config.saveCalculatedFields, []], allEntities: [config.allEntities, []], entityIds: [config.entityIds, [Validators.required]] }) @@ -202,6 +206,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements saveAttributes: true, saveRelations: true, saveCredentials: true, + saveCalculatedFields: true, allEntities: true, entityIds: [] }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index 4736bb1ee2..b877d19b63 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -72,6 +72,9 @@ {{ 'version-control.load-relations' | translate }} + + {{ 'version-control.load-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 23f09ae977..f06abaf3a8 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -31,7 +31,8 @@ import { PageComponent } from '@shared/components/page.component'; import { entityTypesWithoutRelatedData, EntityTypeVersionLoadConfig, - exportableEntityTypes + exportableEntityTypes, + typesWithCalculatedFields } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -75,6 +76,8 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loading = true; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private translate: TranslateService, private popoverService: TbPopoverService, @@ -145,6 +148,7 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loadRelations: [config.loadRelations, []], loadAttributes: [config.loadAttributes, []], loadCredentials: [config.loadCredentials, []], + loadCalculatedFields: [config.loadCalculatedFields, []], removeOtherEntities: [config.removeOtherEntities, []], findExistingEntityByName: [config.findExistingEntityByName, []] }) @@ -180,6 +184,7 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loadAttributes: true, loadRelations: true, loadCredentials: true, + loadCalculatedFields: true, removeOtherEntities: false, findExistingEntityByName: true }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 10c2dc5e40..da4c31a1b8 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -47,6 +47,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 48b2a2a174..0c4b696ad1 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -20,6 +20,7 @@ import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms import { entityTypesWithoutRelatedData, SingleEntityVersionCreateRequest, + typesWithCalculatedFields, VersionCreateRequestType, VersionCreationResult } from '@shared/models/vc.models'; @@ -71,6 +72,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni private versionCreateResultSubscription: Subscription; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, private cd: ChangeDetectorRef, @@ -115,7 +118,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni ? this.createVersionFormGroup.get('saveRelations').value : false, saveAttributes: !entityTypesWithoutRelatedData.has(this.entityId.entityType) ? this.createVersionFormGroup.get('saveAttributes').value : false, - saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false + saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false, + saveCalculatedFields: typesWithCalculatedFields.has(this.entityId.entityType) ? this.createVersionFormGroup.get('saveCalculatedFields').value : false, }, type: VersionCreateRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index 4bc61757c4..1183223850 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -36,6 +36,9 @@ {{ 'version-control.load-relations' | translate }} + + {{ 'version-control.load-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index df2e8c19ff..4456eb7ce7 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -79,7 +79,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn this.restoreFormGroup = this.fb.group({ loadAttributes: [true, []], loadRelations: [true, []], - loadCredentials: [true, []] + loadCredentials: [true, []], + loadCalculatedFields: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; @@ -110,7 +111,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn config: { loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, - loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false, + loadCalculatedFields: this.entityDataInfo.hasCalculatedFields ? this.restoreFormGroup.get('loadCalculatedFields').value : false }, type: VersionLoadRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index 7c7d14eff5..eccb5cb908 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -587,6 +587,10 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal newMode.$highlightRules.$rules[group] = this.highlightRules[group]; } } + const identifierRule = newMode.$highlightRules.$rules.no_regex.find(rule => rule.token?.includes('identifier')); + if (identifierRule) { + identifierRule.next = 'start'; + } // @ts-ignore this.jsEditor.session.$onChangeMode(newMode); } diff --git a/ui-ngx/src/app/shared/models/ace/ace.models.ts b/ui-ngx/src/app/shared/models/ace/ace.models.ts index 5287886b84..49cf670989 100644 --- a/ui-ngx/src/app/shared/models/ace/ace.models.ts +++ b/ui-ngx/src/app/shared/models/ace/ace.models.ts @@ -377,4 +377,3 @@ export const endGroupHighlightRule: AceHighlightRule = { }; - diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index f6d7aaaa1e..7083ce7ae0 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -64,6 +64,7 @@ export interface CalculatedFieldOutput { type: OutputType; name: string; scope?: AttributeScope; + decimalsByDefault?: number; } export enum ArgumentEntityType { @@ -272,7 +273,7 @@ export const CalculatedFieldAttributeValueArgumentAutocomplete = { export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { max: { meta: 'function', - description: 'Computes the maximum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Returns the maximum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -288,7 +289,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, min: { meta: 'function', - description: 'Computes the minimum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Returns the minimum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -304,7 +305,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, mean: { meta: 'function', - description: 'Computes the mean value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the mean value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -318,9 +319,25 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { type: 'number' } }, + avg: { + meta: 'function', + description: 'Computes the average value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The average value, or NaN if applicable', + type: 'number' + } + }, std: { meta: 'function', - description: 'Computes the standard deviation in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the standard deviation of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -336,7 +353,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, median: { meta: 'function', - description: 'Computes the median value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the median value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -352,7 +369,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, count: { meta: 'function', - description: 'Counts values in the list of rolling argument values. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', + description: 'Counts values of the rolling argument. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', args: [ { name: 'ignoreNaN', @@ -368,7 +385,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, last: { meta: 'function', - description: 'Returns the last non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the last value.', + description: 'Returns the last non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the last value.', args: [ { name: 'ignoreNaN', @@ -384,7 +401,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, first: { meta: 'function', - description: 'Returns the first non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the first value.', + description: 'Returns the first non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the first value.', args: [ { name: 'ignoreNaN', @@ -400,7 +417,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, sum: { meta: 'function', - description: 'Computes the sum of values in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the sum of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -413,12 +430,56 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { description: 'The sum of values, or NaN if applicable', type: 'number' } + }, + merge: { + meta: 'function', + description: 'Merges current object with other time series rolling argument into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'other', + description: "A time series rolling argument to be merged with the current object.", + type: "object", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } + }, + mergeAll: { + meta: 'function', + description: 'Merges current object with other time series rolling arguments into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'others', + description: "A list of time series rolling arguments to be merged with the current object.", + type: "object[]", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } } }; export const CalculatedFieldRollingValueArgumentAutocomplete = { meta: 'object', - type: '{ values: { ts: number; value: any; }[]; timeWindow: { startTs: number; endTs: number; limit: number } }; }', + type: '{ values: { ts: number; value: number; }[]; timeWindow: { startTs: number; endTs: number } }; }', description: 'Calculated field rolling value argument.', children: { ...CalculatedFieldRollingValueArgumentFunctionsAutocomplete, @@ -429,7 +490,7 @@ export const CalculatedFieldRollingValueArgumentAutocomplete = { }, timeWindow: { meta: 'object', - type: '{ startTs: number; endTs: number; limit: number }', + type: '{ startTs: number; endTs: number }', description: 'Time window configuration', children: { startTs: { @@ -441,11 +502,6 @@ export const CalculatedFieldRollingValueArgumentAutocomplete = { meta: 'number', type: 'number', description: 'End time stamp', - }, - limit: { - meta: 'number', - type: 'number', - description: 'Limit', } } } @@ -504,7 +560,7 @@ const calculatedFieldSingleArgumentValueHighlightRules: AceHighlightRules = { } const calculatedFieldRollingArgumentValueFunctionsHighlightRules: Array = - ['max', 'min', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum'].map(funcName => ({ + ['max', 'min', 'avg', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum', 'merge', 'mergeAll'].map(funcName => ({ token: 'tb.calculated-field-func', regex: `\\b${funcName}\\b`, next: 'no_regex' @@ -549,3 +605,8 @@ const calculatedFieldTimeWindowArgumentValueHighlightRules: AceHighlightRules = endGroupHighlightRule ] } + +export const calculatedFieldDefaultScript = 'return {\n' + + ' // Convert Fahrenheit to Celsius\n' + + ' "temperatureCelsius": (temperature.value - 32) / 1.8\n' + + '};' diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts index f0489060ef..b8b1be4e83 100644 --- a/ui-ngx/src/app/shared/models/regex.constants.ts +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -14,6 +14,8 @@ /// limitations under the License. /// -export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; +export const oneSpaceInsideRegex = /^\s*\S+(?:\s\S+)*\s*$/; -export const charsWithNumRegex = /^[a-zA-Z]+[a-zA-Z0-9]*$/; +export const charsWithNumRegex = /^[a-zA-Z_]+[a-zA-Z0-9_]*$/; + +export const digitsRegex = /^\d*$/; diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 38ff63e3c6..23896141db 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -156,11 +156,11 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan rpcTtlDays: 0, queueStatsTtlDays: 0, ruleEngineExceptionsTtlDays: 0, - maxCalculatedFieldsPerEntity: 0, - maxArgumentsPerCF: 0, - maxDataPointsPerRollingArg: 0, - maxStateSizeInKBytes: 0, - maxSingleValueArgumentSizeInKBytes: 0, + maxCalculatedFieldsPerEntity: 5, + maxArgumentsPerCF: 10, + maxDataPointsPerRollingArg: 1000, + maxStateSizeInKBytes: 32, + maxSingleValueArgumentSizeInKBytes: 2, calculatedFieldDebugEventsRateLimit: '' }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 6cc8de02eb..3795518ffc 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -49,6 +49,7 @@ export interface VersionCreateConfig { saveRelations: boolean; saveAttributes: boolean; saveCredentials: boolean; + saveCalculatedFields: boolean; } export enum VersionCreateRequestType { @@ -106,6 +107,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: syncStrategy: null, saveAttributes: !entityTypesWithoutRelatedData.has(entityType), saveRelations: !entityTypesWithoutRelatedData.has(entityType), + saveCalculatedFields: typesWithCalculatedFields.has(entityType), saveCredentials: true, allEntities: true, entityIds: [] @@ -118,6 +120,7 @@ export interface VersionLoadConfig { loadRelations: boolean; loadAttributes: boolean; loadCredentials: boolean; + loadCalculatedFields: boolean; } export enum VersionLoadRequestType { @@ -154,6 +157,7 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En loadAttributes: !entityTypesWithoutRelatedData.has(entityType), loadRelations: !entityTypesWithoutRelatedData.has(entityType), loadCredentials: true, + loadCalculatedFields: typesWithCalculatedFields.has(entityType), removeOtherEntities: false, findExistingEntityByName: true }; @@ -254,4 +258,7 @@ export interface EntityDataInfo { hasRelations: boolean; hasAttributes: boolean; hasCredentials: boolean; + hasCalculatedFields: boolean; } + +export const typesWithCalculatedFields = new Set([EntityType.DEVICE, EntityType.ASSET, EntityType.ASSET_PROFILE, EntityType.DEVICE_PROFILE]); diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 02c604d8ae..a26f267c64 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1015,6 +1015,7 @@ "script": "Script" }, "arguments": "Arguments", + "decimals-by-default": "Decimals by default", "debugging": "Calculated field debugging", "argument-name": "Argument name", "datasource": "Datasource", @@ -1031,6 +1032,7 @@ "argument-type": "Argument type", "see-debug-events": "See debug events", "attribute": "Attribute", + "copy-argument-name": "Copy argument name", "timeseries-key": "Time series key", "device-name": "Device name", "latest-telemetry": "Latest telemetry", @@ -1070,6 +1072,7 @@ "argument-name-max-length": "Argument name should be less than 256 characters.", "argument-type-required": "Argument type is required.", "max-args": "Maximum number of arguments reached.", + "decimals-range": "Decimals by default should be a number between 0 and 15.", "expression": "Default expression demonstrates how to transform a temperature from Fahrenheit to Celsius." } }, @@ -2521,8 +2524,8 @@ "type-current-tenant": "Current Tenant", "type-current-user": "Current User", "type-current-user-owner": "Current User Owner", - "type-calculated-field": "Calculated Field", - "type-calculated-fields": "Calculated Fields", + "type-calculated-field": "Calculated field", + "type-calculated-fields": "Calculated fields", "type-widgets-bundle": "Widgets bundle", "type-widgets-bundles": "Widgets bundles", "list-of-widgets-bundles": "{ count, plural, =1 {One widgets bundle} other {List of # widget bundles} }", @@ -5534,21 +5537,21 @@ "tenant-entity-import-rate-limit": "Entity version load", "tenant-notification-request-rate-limit": "Notification requests", "tenant-notification-requests-per-rule-rate-limit": "Notification requests per notification rule", - "max-calculated-fields": "Maximum number of calculated fields per entity", - "max-calculated-fields-range": "Maximum number of calculated fields per entity can't be negative", - "max-calculated-fields-required": "Maximum number of calculated fields per entity is required", - "max-data-points-per-rolling-arg": "Maximum number of data points in a time series rolling arguments", - "max-data-points-per-rolling-arg-range": "Maximum number of data points in a time series rolling arguments can't be negative", - "max-data-points-per-rolling-arg-required": "Maximum number of data points in a time series rolling arguments is required", - "max-arguments-per-cf": "Maximum number of arguments per calculated field", - "max-arguments-per-cf-range": "Maximum number of arguments per calculated field can't be negative", - "max-arguments-per-cf-required": "Maximum number of arguments per calculated field is required", - "max-state-size": "Maximum size of the state in KB", - "max-state-size-range": "Maximum size of the state in KB can't be negative", - "max-state-size-required": "Maximum size of the state in KB is required", - "max-value-argument-size": "Maximum size of the single value argument in KB", - "max-value-argument-size-range": "Maximum size of the single value argument in KB can't be negative", - "max-value-argument-size-required": "Maximum size of the single value argument in KB is required", + "max-calculated-fields": "Calculated fields per entity maximum number", + "max-calculated-fields-range": "Calculated fields per entity maximum number can't be negative", + "max-calculated-fields-required": "Calculated fields per entity maximum number is required", + "max-data-points-per-rolling-arg": "Maximum data points number in rolling arguments", + "max-data-points-per-rolling-arg-range": "Maximum data points number in rolling arguments can't be negative", + "max-data-points-per-rolling-arg-required": "Maximum data points number in rolling arguments is required", + "max-arguments-per-cf": "Arguments per calculated field maximum number", + "max-arguments-per-cf-range": "Arguments per calculated field maximum number can't be negative", + "max-arguments-per-cf-required": "Arguments per calculated field maximum number is required", + "max-state-size": "State maximum size in KB", + "max-state-size-range": "State maximum size in KB can't be negative", + "max-state-size-required": "State maximum size in KB is required", + "max-value-argument-size": "Single value argument maximum size in KB", + "max-value-argument-size-range": "Single value argument maximum size in KB can't be negative", + "max-value-argument-size-required": "Single value argument maximum size in KB is required", "max-transport-messages": "Transport messages maximum number", "max-transport-messages-required": "Transport messages maximum number is required.", "max-transport-messages-range": "Transport messages maximum number can't be negative", @@ -6320,6 +6323,7 @@ "export-relations": "Export relations", "export-attributes": "Export attributes", "export-credentials": "Export credentials", + "export-calculated-fields": "Export calculated fields", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -6336,6 +6340,7 @@ "load-relations": "Load relations", "load-attributes": "Load attributes", "load-credentials": "Load credentials", + "load-calculated-fields": "Load calculated fields", "compare-with-current": "Compare with current", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference",