Browse Source

Merge branch 'rc' into release-4.2

release-4.2
Igor Kulikov 10 months ago
parent
commit
0b3cab5899
  1. 4
      application/pom.xml
  2. 15
      application/src/main/data/json/system/widget_types/markdown_html_card.json
  3. 2
      application/src/main/data/resources/dashboards/gateways_dashboard.json
  4. 30
      application/src/main/data/upgrade/basic/schema_update.sql
  5. 5
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  6. 3
      application/src/main/java/org/thingsboard/server/actors/app/AppActor.java
  7. 12
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java
  8. 100
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
  9. 19
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  10. 3
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  11. 1
      application/src/main/java/org/thingsboard/server/controller/ImageController.java
  12. 17
      application/src/main/java/org/thingsboard/server/controller/TbResourceController.java
  13. 42
      application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java
  14. 40
      application/src/main/java/org/thingsboard/server/service/ai/Langchain4jChatModelConfigurerImpl.java
  15. 4
      application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java
  16. 8
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java
  17. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  18. 30
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java
  19. 2
      application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java
  20. 2
      application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java
  21. 4
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java
  22. 9
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  23. 4
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  24. 2
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java
  25. 4
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  26. 5
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  27. 4
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java
  28. 1
      application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java
  29. 10
      application/src/main/resources/thingsboard.yml
  30. 53
      application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
  31. 12
      application/src/test/java/org/thingsboard/server/controller/AiModelControllerTest.java
  32. 63
      application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java
  33. 23
      application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java
  34. 50
      application/src/test/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateServiceTest.java
  35. 2
      application/src/test/java/org/thingsboard/server/service/entitiy/ai/DefaultTbAiModelServiceTest.java
  36. 83
      application/src/test/java/org/thingsboard/server/service/resource/DefaultResourceDataCacheTest.java
  37. 125
      application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java
  38. 39
      application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java
  39. 11
      application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesJsonIntegrationTest.java
  40. 25
      application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java
  41. 5
      common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java
  42. 20
      common/dao-api/src/main/java/org/thingsboard/server/dao/resource/TbResourceDataCache.java
  43. 25
      common/data/src/main/java/org/thingsboard/server/common/data/GeneralFileDescriptor.java
  44. 3
      common/data/src/main/java/org/thingsboard/server/common/data/ResourceType.java
  45. 7
      common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java
  46. 6
      common/data/src/main/java/org/thingsboard/server/common/data/TbResource.java
  47. 21
      common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDataInfo.java
  48. 3
      common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java
  49. 6
      common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbChatResponse.java
  50. 8
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/AiModelConfig.java
  51. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AiChatModelConfig.java
  52. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AmazonBedrockChatModelConfig.java
  53. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AnthropicChatModelConfig.java
  54. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AzureOpenAiChatModelConfig.java
  55. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/GitHubModelsChatModelConfig.java
  56. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/GoogleAiGeminiChatModelConfig.java
  57. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/GoogleVertexAiGeminiChatModelConfig.java
  58. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/Langchain4jChatModelConfigurer.java
  59. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/MistralAiChatModelConfig.java
  60. 58
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/OllamaChatModelConfig.java
  61. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/OpenAiChatModelConfig.java
  62. 3
      common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/AiProvider.java
  63. 2
      common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/AiProviderConfig.java
  64. 48
      common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/OllamaProviderConfig.java
  65. 30
      common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/OpenAiProviderConfig.java
  66. 2
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java
  67. 3
      common/data/src/main/java/org/thingsboard/server/common/data/mobile/bundle/MobileAppBundle.java
  68. 1
      common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRule.java
  69. 2
      common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleConfig.java
  70. 3
      common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java
  71. 12
      common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java
  72. 246
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java
  73. 27
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java
  74. 15
      common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java
  75. 5
      dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java
  76. 5
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
  77. 72
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java
  78. 72
      dao/src/main/java/org/thingsboard/server/dao/resource/DefaultTbResourceDataCache.java
  79. 2
      dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceDao.java
  80. 2
      dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceInfoDao.java
  81. 4
      dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java
  82. 6
      dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java
  83. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java
  84. 16
      dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java
  85. 10
      dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java
  86. 6
      dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java
  87. 8
      dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceInfoDao.java
  88. 6
      dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java
  89. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceRepository.java
  90. 12
      dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java
  91. 15
      dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java
  92. 10
      dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java
  93. 15
      dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java
  94. 29
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  95. 15
      dao/src/main/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDate.java
  96. 1
      dao/src/test/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDaoTest.java
  97. 12
      dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMinutesAlwaysExistsTest.java
  98. 63
      dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest.java
  99. 12
      dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningYearsAlwaysExistsTest.java
  100. 41
      dao/src/test/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDateTest.java

4
application/pom.xml

@ -419,6 +419,10 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-ollama</artifactId>
</dependency>
</dependencies>
<build>

15
application/src/main/data/json/system/widget_types/markdown_html_card.json

File diff suppressed because one or more lines are too long

2
application/src/main/data/resources/dashboards/gateways_dashboard.json

@ -650,7 +650,7 @@
"settings": {
"useMarkdownTextFunction": true,
"markdownTextPattern": "# Markdown/HTML card \\n - **Current entity**: **${entityName}**. \\n - **Current value**: **${Random}**.",
"markdownTextFunction": "var blockData = '';\nvar connectorsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Connectors\");\nvar logsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Logs\");\nfunction generateMatHeader(index) {\n if (index !== undefined && index > -1) {\n return `<mat-card-header class='tb-home-widget-link' (click)=\"ctx.actionsApi.handleWidgetAction($event, ctx.actionsApi.getActionDescriptors('elementClick')[${index}], ctx.datasources[0].entity.id)\">`\n } else {\n return \"<mat-card-header>\"\n }\n}\nfunction createDataBlock(value, label, dividerStyle, mobile, index) {\n blockData += `\n <mat-card style=\"flex-grow: 1; width: ${mobile ? '100%' : 'auto'}; min-height: ${mobile ? 'auto' : '57px'}\" class=\"${dividerStyle}\">\n <div class=\"divider\"></div>\n <mat-divider vertical style=\"height:100%\"></mat-divider>\n ${generateMatHeader(index)}\n <mat-card-subtitle>${label}</mat-card-subtitle>\n </mat-card-header>\n <mat-card-content> ${value}</mat-card-content>\n </mat-card>`;\n}\ncreateDataBlock(data[0].Status, \"Status\", data[0].Status === \"Active\" ? 'divider-green' : 'divider-red');\ncreateDataBlock(data[0].Name, \"Gateway Name\", '', ctx.isMobile);\nif (data[0].Version) {\n createDataBlock(data[0].Version, \"Gateway Version\", '');\n}\ncreateDataBlock(data[0].Type, \"Gateway Type\", '');\ncreateDataBlock(\n `<span style=\"color:rgb(25,128,56)\">${(data[1] ? data[1].count : 0)} </span>`\n + \" | \" +\n `<span style=\"color:rgb(203,37,48)\">${(data[2] ? data[2][\"count 2\"] : 0)} </span>`\n , \"Devices <span class='tb-hint' style='padding-left: 0'>(Active | Inactive)</span>\", '');\ncreateDataBlock(\n `<span style=\"color:rgb(25,128,56)\">${(data[0].active_connectors ? JSON.parse(data[0].active_connectors).length : 0)} </span>`\n + \" | \" +\n `<span style=\"color:rgb(203,37,48)\">${(data[0].inactive_connectors ? JSON.parse(data[0].inactive_connectors).length : 0)} </span>`\n , \"Connectors <span class='tb-hint' style='padding-left: 0'>(Enabled | Disabled)</span>\", '', '', connectorsIndex);\ncreateDataBlock(data[0].ALL_ERRORS_COUNT || 0, \"Errors\", (data[0].ALL_ERRORS_COUNT || 0) === 0 ? 'divider-green' : 'divider-red', '', logsIndex);\nreturn `<div class=\"flex flex-row flex-wrap gap-2 cards-container\">${blockData}</div>`;",
"markdownTextFunction": "var blockData = '';\nvar connectorsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Connectors\");\nvar logsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Logs\");\nfunction generateMatHeader(index) {\n if (index !== undefined && index > -1) {\n return `<mat-card-header class='tb-home-widget-link' (click)=\"ctx.actionsApi.handleWidgetAction($event, ctx.actionsApi.getActionDescriptors('elementClick')[${index}], ctx.datasources[0].entity.id)\">`\n } else {\n return \"<mat-card-header>\"\n }\n}\nfunction createDataBlock(value, label, dividerStyle, mobile, index) {\n blockData += `\n <mat-card style=\"flex-grow: 1; width: ${mobile ? '100%' : 'auto'}; min-height: ${mobile ? 'auto' : '57px'}\" class=\"${dividerStyle}\">\n <div class=\"divider\"></div>\n <mat-divider vertical style=\"height:100%\"></mat-divider>\n ${generateMatHeader(index)}\n <mat-card-subtitle>${label}</mat-card-subtitle>\n </mat-card-header>\n <mat-card-content> ${ctx.sanitizer.sanitize(1, value)}</mat-card-content>\n </mat-card>`;\n}\ncreateDataBlock(data[0].Status, \"Status\", data[0].Status === \"Active\" ? 'divider-green' : 'divider-red');\ncreateDataBlock(data[0].Name, \"Gateway Name\", '', ctx.isMobile);\nif (data[0].Version) {\n createDataBlock(data[0].Version, \"Gateway Version\", '');\n}\ncreateDataBlock(data[0].Type, \"Gateway Type\", '');\ncreateDataBlock(\n `<span style=\"color:rgb(25,128,56)\">${(data[1] ? data[1].count : 0)} </span>`\n + \" | \" +\n `<span style=\"color:rgb(203,37,48)\">${(data[2] ? data[2][\"count 2\"] : 0)} </span>`\n , \"Devices <span class='tb-hint' style='padding-left: 0'>(Active | Inactive)</span>\", '');\ncreateDataBlock(\n `<span style=\"color:rgb(25,128,56)\">${(data[0].active_connectors ? JSON.parse(data[0].active_connectors).length : 0)} </span>`\n + \" | \" +\n `<span style=\"color:rgb(203,37,48)\">${(data[0].inactive_connectors ? JSON.parse(data[0].inactive_connectors).length : 0)} </span>`\n , \"Connectors <span class='tb-hint' style='padding-left: 0'>(Enabled | Disabled)</span>\", '', '', connectorsIndex);\ncreateDataBlock(data[0].ALL_ERRORS_COUNT || 0, \"Errors\", (data[0].ALL_ERRORS_COUNT || 0) === 0 ? 'divider-green' : 'divider-red', '', logsIndex);\nreturn `<div class=\"flex flex-row flex-wrap gap-2 cards-container\">${blockData}</div>`;",
"applyDefaultMarkdownStyle": false,
"markdownCss": ".divider {\n position: absolute;\n width: 3px;\n top: 8px;\n border-radius: 2px;\n bottom: 8px;\n border: 1px solid rgba(31, 70, 144, 1);\n background-color: rgba(31, 70, 144, 1);\n left: 10px;\n}\n.divider-green .divider {\n border: 1px solid rgb(25,128,56);\n background-color: rgb(25,128,56);\n}\n\n.divider-green .mat-mdc-card-content {\n color: rgb(25,128,56);\n}\n\n.divider-red .divider {\n border: 1px solid rgb(203,37,48);\n background-color: rgb(203,37,48);\n}\n\n.divider-red .mat-mdc-card-content {\n color: rgb(203,37,48);\n}\n\n.mdc-card {\n position: relative;\n padding-left: 10px;\n margin-bottom: 1px;\n}\n\n.mat-mdc-card-subtitle {\n font-weight: 400;\n font-size: 12px;\n}\n\n.mat-mdc-card-header {\n padding: 8px 16px 0;\n}\n\n.mat-mdc-card-content:last-child {\n padding-bottom: 8px;\n font-size: 16px;\n}\n\n.cards-container {\n height: calc(100% - 1px);\n justify-content: stretch;\n align-items: center;\n margin-bottom: 1px;\n}\n\n::ng-deep.tb-home-widget-link > div {\n flex-grow: 1;\n cursor: pointer;\n}\n\n .tb-home-widget-link {\n width: 100%;\n }\n\n .tb-home-widget-link:hover::after{\n color: inherit;\n }\n \n .tb-home-widget-link::after{\n content: 'arrow_forward';\n display: inline-block;\n transform: rotate(315deg);\n font-family: 'Material Icons';\n font-weight: normal;\n font-style: normal;\n font-size: 18px;\n color: rgba(0, 0, 0, 0.12);\n vertical-align: bottom;\n margin-left: 6px;\n}"
},

30
application/src/main/data/upgrade/basic/schema_update.sql

@ -14,33 +14,3 @@
-- limitations under the License.
--
-- UPDATE OTA PACKAGE EXTERNAL ID START
ALTER TABLE ota_package
ADD COLUMN IF NOT EXISTS external_id uuid;
DO
$$
BEGIN
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'ota_package_external_id_unq_key') THEN
ALTER TABLE ota_package ADD CONSTRAINT ota_package_external_id_unq_key UNIQUE (tenant_id, external_id);
END IF;
END;
$$;
-- UPDATE OTA PACKAGE EXTERNAL ID END
-- DROP INDEXES THAT DUPLICATE UNIQUE CONSTRAINT START
DROP INDEX IF EXISTS idx_device_external_id;
DROP INDEX IF EXISTS idx_device_profile_external_id;
DROP INDEX IF EXISTS idx_asset_external_id;
DROP INDEX IF EXISTS idx_entity_view_external_id;
DROP INDEX IF EXISTS idx_rule_chain_external_id;
DROP INDEX IF EXISTS idx_dashboard_external_id;
DROP INDEX IF EXISTS idx_customer_external_id;
DROP INDEX IF EXISTS idx_widgets_bundle_external_id;
-- DROP INDEXES THAT DUPLICATE UNIQUE CONSTRAINT END
ALTER TABLE mobile_app ADD COLUMN IF NOT EXISTS title varchar(255);

5
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -97,6 +97,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.queue.QueueStatsService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.rule.RuleNodeStateService;
@ -511,6 +512,10 @@ public class ActorSystemContext {
@Getter
private ResourceService resourceService;
@Autowired
@Getter
private TbResourceDataCache resourceDataCache;
@Lazy
@Autowired(required = false)
@Getter

3
application/src/main/java/org/thingsboard/server/actors/app/AppActor.java

@ -114,9 +114,6 @@ public class AppActor extends ContextAwareActor {
ctx.broadcastToChildrenByType(msg, EntityType.TENANT);
break;
case CF_CACHE_INIT_MSG:
case CF_INIT_PROFILE_ENTITY_MSG:
case CF_INIT_MSG:
case CF_LINK_INIT_MSG:
case CF_STATE_RESTORE_MSG:
//TODO: use priority from the message body. For example, messages about CF lifecycle are important and Device lifecycle are not.
// same for the Linked telemetry.

12
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java

@ -24,9 +24,6 @@ import org.thingsboard.server.common.msg.TbActorStopReason;
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitProfileEntityMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg;
/**
@ -70,15 +67,6 @@ public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor {
case CF_CACHE_INIT_MSG:
processor.onCacheInitMsg((CalculatedFieldCacheInitMsg) msg);
break;
case CF_INIT_PROFILE_ENTITY_MSG:
processor.onProfileEntityMsg((CalculatedFieldInitProfileEntityMsg) msg);
break;
case CF_INIT_MSG:
processor.onFieldInitMsg((CalculatedFieldInitMsg) msg);
break;
case CF_LINK_INIT_MSG:
processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg);
break;
case CF_STATE_RESTORE_MSG:
processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg);
break;

100
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java

@ -35,9 +35,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitProfileEntityMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
@ -120,37 +117,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
msg.getCallback().onSuccess();
}
public void onProfileEntityMsg(CalculatedFieldInitProfileEntityMsg msg) {
log.debug("[{}] Processing profile entity message.", msg.getTenantId().getId());
entityProfileCache.add(msg.getProfileEntityId(), msg.getEntityId());
msg.getCallback().onSuccess();
}
public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException {
log.debug("[{}] Processing CF init message.", msg.getCf().getId());
var cf = msg.getCf();
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService());
try {
cfCtx.init();
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build();
}
calculatedFields.put(cf.getId(), cfCtx);
// We use copy on write lists to safely pass the reference to another actor for the iteration.
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx);
msg.getCallback().onSuccess();
}
public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) {
log.debug("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId());
var link = msg.getLink();
// We use copy on write lists to safely pass the reference to another actor for the iteration.
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link);
msg.getCallback().onSuccess();
}
public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) {
var cfId = msg.getId().cfId();
var calculatedField = calculatedFields.get(cfId);
@ -321,29 +287,29 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
newCfCtx.init();
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build();
}
calculatedFields.put(newCf.getId(), newCfCtx);
List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId());
List<CalculatedFieldCtx> newCfList = new CopyOnWriteArrayList<>();
boolean found = false;
for (CalculatedFieldCtx oldCtx : oldCfList) {
if (oldCtx.getCfId().equals(newCf.getId())) {
} finally {
calculatedFields.put(newCf.getId(), newCfCtx);
List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId());
List<CalculatedFieldCtx> newCfList = new CopyOnWriteArrayList<>();
boolean found = false;
for (CalculatedFieldCtx oldCtx : oldCfList) {
if (oldCtx.getCfId().equals(newCf.getId())) {
newCfList.add(newCfCtx);
found = true;
} else {
newCfList.add(oldCtx);
}
}
if (!found) {
newCfList.add(newCfCtx);
found = true;
} else {
newCfList.add(oldCtx);
}
// We use copy on write lists to safely pass the reference to another actor for the iteration.
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
entityIdCalculatedFields.put(newCf.getEntityId(), newCfList);
deleteLinks(oldCfCtx);
addLinks(newCf);
}
if (!found) {
newCfList.add(newCfCtx);
}
entityIdCalculatedFields.put(newCf.getEntityId(), newCfList);
deleteLinks(oldCfCtx);
addLinks(newCf);
// We use copy on write lists to safely pass the reference to another actor for the iteration.
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
var stateChanges = newCfCtx.hasStateChanges(oldCfCtx);
if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) {
initCf(newCfCtx, callback, stateChanges);
@ -566,20 +532,38 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
cfs.forEach(cf -> {
log.trace("Processing calculated field record: {}", cf);
try {
onFieldInitMsg(new CalculatedFieldInitMsg(cf.getTenantId(), cf));
initCalculatedField(cf);
} catch (CalculatedFieldException e) {
log.error("Failed to process calculated field record: {}", cf, e);
}
});
calculatedFields.values().forEach(cf -> {
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf);
});
PageDataIterable<CalculatedFieldLink> cfls = new PageDataIterable<>(pageLink -> cfDaoService.findAllCalculatedFieldLinksByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize());
cfls.forEach(link -> {
onLinkInitMsg(new CalculatedFieldLinkInitMsg(link.getTenantId(), link));
log.trace("Processing calculated field link record: {}", link);
initCalculatedFieldLink(link);
});
}
private void initCalculatedField(CalculatedField cf) throws CalculatedFieldException {
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService());
try {
cfCtx.init();
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build();
} finally {
calculatedFields.put(cf.getId(), cfCtx);
// We use copy on write lists to safely pass the reference to another actor for the iteration.
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx);
}
}
private void initCalculatedFieldLink(CalculatedFieldLink link) {
// We use copy on write lists to safely pass the reference to another actor for the iteration.
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link);
}
private void initEntityProfileCache() {
PageDataIterable<ProfileEntityIdInfo> deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize());
for (ProfileEntityIdInfo idInfo : deviceIdInfos) {

19
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasRuleEngineProfile;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.alarm.Alarm;
@ -60,6 +61,7 @@ import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
@ -110,6 +112,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.queue.QueueStatsService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantService;
@ -770,6 +773,11 @@ public class DefaultTbContext implements TbContext {
return mainCtx.getResourceService();
}
@Override
public TbResourceDataCache getTbResourceDataCache() {
return mainCtx.getResourceDataCache();
}
@Override
public OtaPackageService getOtaPackageService() {
return mainCtx.getOtaPackageService();
@ -1054,11 +1062,20 @@ public class DefaultTbContext implements TbContext {
@Override
public void checkTenantEntity(EntityId entityId) throws TbNodeException {
if (!this.getTenantId().equals(TenantIdLoader.findTenantId(this, entityId))) {
TenantId actualTenantId = TenantIdLoader.findTenantId(this, entityId);
if (!getTenantId().equals(actualTenantId)) {
throw new TbNodeException("Entity with id: '" + entityId + "' specified in the configuration doesn't belong to the current tenant.", true);
}
}
@Override
public <E extends HasId<I> & HasTenantId, I extends EntityId> void checkTenantOrSystemEntity(E entity) throws TbNodeException {
TenantId actualTenantId = entity.getTenantId();
if (!getTenantId().equals(actualTenantId) && !actualTenantId.isSysTenantId()) {
throw new TbNodeException("Entity with id: '" + entity.getId() + "' specified in the configuration doesn't belong to the current or system tenant.", true);
}
}
private static String getFailureMessage(Throwable th) {
String failureMessage;
if (th != null) {

3
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java

@ -180,9 +180,6 @@ public class TenantActor extends RuleChainManagerActor {
onRuleChainMsg((RuleChainAwareMsg) msg);
break;
case CF_CACHE_INIT_MSG:
case CF_INIT_PROFILE_ENTITY_MSG:
case CF_INIT_MSG:
case CF_LINK_INIT_MSG:
case CF_STATE_RESTORE_MSG:
case CF_PARTITIONS_CHANGE_MSG:
onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true);

1
application/src/main/java/org/thingsboard/server/controller/ImageController.java

@ -300,6 +300,7 @@ public class ImageController extends BaseController {
tbImageService.putETag(cacheKey, descriptor.getEtag());
var result = ResponseEntity.ok()
.header("Content-Type", descriptor.getMediaType())
.header("Content-Security-Policy", "default-src 'none'")
.eTag(descriptor.getEtag());
if (!cacheKey.isPublic()) {
result

17
application/src/main/java/org/thingsboard/server/controller/TbResourceController.java

@ -55,14 +55,17 @@ import org.thingsboard.server.common.data.util.ThrowingSupplier;
import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.resource.TbResourceService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import static org.thingsboard.server.controller.ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER;
import static org.thingsboard.server.controller.ControllerConstants.LWM2M_OBJECT_DESCRIPTION;
@ -263,6 +266,20 @@ public class TbResourceController extends BaseController {
}
}
@ApiOperation(value = "Get Resource Infos by ids (getSystemOrTenantResourcesByIds)")
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@GetMapping(value = "/resource", params = {"resourceIds"})
public List<TbResourceInfo> getSystemOrTenantResourcesByIds(
@Parameter(description = "A list of resource ids, separated by comma ','", array = @ArraySchema(schema = @Schema(type = "string")))
@RequestParam("resourceIds") Set<UUID> resourceUuids) throws ThingsboardException {
SecurityUser user = getCurrentUser();
List<TbResourceId> resourceIds = new ArrayList<>();
for (UUID resourceId : resourceUuids) {
resourceIds.add(new TbResourceId(resourceId));
}
return resourceService.findSystemOrTenantResourcesByIds(user.getTenantId(), resourceIds);
}
@ApiOperation(value = "Get All Resource Infos (getAllResources)",
notes = "Returns a page of Resource Info objects owned by tenant. " +
PAGE_DATA_PARAMETERS + RESOURCE_INFO_DESCRIPTION + TENANT_AUTHORITY_PARAGRAPH)

42
application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java

@ -16,6 +16,11 @@
package org.thingsboard.server.service.ai;
import com.google.common.util.concurrent.FluentFuture;
import dev.langchain4j.data.message.ChatMessage;
import dev.langchain4j.data.message.Content;
import dev.langchain4j.data.message.TextContent;
import dev.langchain4j.data.message.UserMessage;
import dev.langchain4j.model.ModelProvider;
import dev.langchain4j.model.chat.ChatModel;
import dev.langchain4j.model.chat.request.ChatRequest;
import dev.langchain4j.model.chat.response.ChatResponse;
@ -24,6 +29,11 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.ai.model.chat.AiChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConfigurer;
import java.util.List;
import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.StringUtils.escapeControlChars;
@Service
@RequiredArgsConstructor
class AiChatModelServiceImpl implements AiChatModelService {
@ -34,7 +44,39 @@ class AiChatModelServiceImpl implements AiChatModelService {
@Override
public <C extends AiChatModelConfig<C>> FluentFuture<ChatResponse> sendChatRequestAsync(AiChatModelConfig<C> chatModelConfig, ChatRequest chatRequest) {
ChatModel langChainChatModel = chatModelConfig.configure(chatModelConfigurer);
if (langChainChatModel.provider() == ModelProvider.GITHUB_MODELS) {
chatRequest = prepareGithubChatRequest(chatRequest);
}
return aiRequestsExecutor.sendChatRequestAsync(langChainChatModel, chatRequest);
}
private ChatRequest prepareGithubChatRequest(ChatRequest chatRequest) {
List<ChatMessage> messages = chatRequest.messages().stream()
.map(this::prepareUserMessage)
.collect(Collectors.toList());
return ChatRequest.builder()
.messages(messages)
.responseFormat(chatRequest.responseFormat())
.build();
}
private ChatMessage prepareUserMessage(ChatMessage message) {
if (message instanceof UserMessage userMessage) {
List<Content> newContents = userMessage.contents().stream()
.map(this::prepareContent)
.collect(Collectors.toList());
return UserMessage.from(newContents);
}
return message;
}
private Content prepareContent(Content content) {
if (content instanceof TextContent txt) {
return new TextContent(escapeControlChars(txt.text()));
}
return content;
}
}

40
application/src/main/java/org/thingsboard/server/service/ai/Langchain4jChatModelConfigurerImpl.java

@ -32,8 +32,10 @@ import dev.langchain4j.model.chat.request.ChatRequestParameters;
import dev.langchain4j.model.github.GitHubModelsChatModel;
import dev.langchain4j.model.googleai.GoogleAiGeminiChatModel;
import dev.langchain4j.model.mistralai.MistralAiChatModel;
import dev.langchain4j.model.ollama.OllamaChatModel;
import dev.langchain4j.model.openai.OpenAiChatModel;
import dev.langchain4j.model.vertexai.gemini.VertexAiGeminiChatModel;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.ai.model.chat.AmazonBedrockChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.AnthropicChatModelConfig;
@ -43,10 +45,12 @@ import org.thingsboard.server.common.data.ai.model.chat.GoogleAiGeminiChatModelC
import org.thingsboard.server.common.data.ai.model.chat.GoogleVertexAiGeminiChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConfigurer;
import org.thingsboard.server.common.data.ai.model.chat.MistralAiChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.OllamaChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.OpenAiChatModelConfig;
import org.thingsboard.server.common.data.ai.provider.AmazonBedrockProviderConfig;
import org.thingsboard.server.common.data.ai.provider.AzureOpenAiProviderConfig;
import org.thingsboard.server.common.data.ai.provider.GoogleVertexAiGeminiProviderConfig;
import org.thingsboard.server.common.data.ai.provider.OllamaProviderConfig;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
@ -54,7 +58,11 @@ import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Base64;
import static java.util.Collections.singletonMap;
@Component
class Langchain4jChatModelConfigurerImpl implements Langchain4jChatModelConfigurer {
@ -62,6 +70,7 @@ class Langchain4jChatModelConfigurerImpl implements Langchain4jChatModelConfigur
@Override
public ChatModel configureChatModel(OpenAiChatModelConfig chatModelConfig) {
return OpenAiChatModel.builder()
.baseUrl(chatModelConfig.providerConfig().baseUrl())
.apiKey(chatModelConfig.providerConfig().apiKey())
.modelName(chatModelConfig.modelId())
.temperature(chatModelConfig.temperature())
@ -134,7 +143,7 @@ class Langchain4jChatModelConfigurerImpl implements Langchain4jChatModelConfigur
// set request timeout from model config
if (chatModelConfig.timeoutSeconds() != null) {
retrySettings.setTotalTimeout(org.threeten.bp.Duration.ofSeconds(chatModelConfig.timeoutSeconds()));
retrySettings.setTotalTimeoutDuration(Duration.ofSeconds(chatModelConfig.timeoutSeconds()));
}
// set updated retry settings
@ -262,6 +271,35 @@ class Langchain4jChatModelConfigurerImpl implements Langchain4jChatModelConfigur
.build();
}
@Override
public ChatModel configureChatModel(OllamaChatModelConfig chatModelConfig) {
var builder = OllamaChatModel.builder()
.baseUrl(chatModelConfig.providerConfig().baseUrl())
.modelName(chatModelConfig.modelId())
.temperature(chatModelConfig.temperature())
.topP(chatModelConfig.topP())
.topK(chatModelConfig.topK())
.numCtx(chatModelConfig.contextLength())
.numPredict(chatModelConfig.maxOutputTokens())
.timeout(toDuration(chatModelConfig.timeoutSeconds()))
.maxRetries(chatModelConfig.maxRetries());
var auth = chatModelConfig.providerConfig().auth();
if (auth instanceof OllamaProviderConfig.OllamaAuth.Basic basicAuth) {
String credentials = basicAuth.username() + ":" + basicAuth.password();
String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8));
builder.customHeaders(singletonMap(HttpHeaders.AUTHORIZATION, "Basic " + encodedCredentials));
} else if (auth instanceof OllamaProviderConfig.OllamaAuth.Token tokenAuth) {
builder.customHeaders(singletonMap(HttpHeaders.AUTHORIZATION, "Bearer " + tokenAuth.token()));
} else if (auth instanceof OllamaProviderConfig.OllamaAuth.None) {
// do nothing
} else {
throw new UnsupportedOperationException("Unknown authentication type: " + auth.getClass().getSimpleName());
}
return builder.build();
}
private static Duration toDuration(Integer timeoutSeconds) {
return timeoutSeconds != null ? Duration.ofSeconds(timeoutSeconds) : null;
}

4
application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java

@ -442,13 +442,13 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService
boolean check(long threshold, long warnThreshold, long value);
}
private void checkStartOfNextCycle() {
public void checkStartOfNextCycle() {
updateLock.lock();
try {
long now = System.currentTimeMillis();
myUsageStates.values().forEach(state -> {
if ((state.getNextCycleTs() < now) && (now - state.getNextCycleTs() < TimeUnit.HOURS.toMillis(1))) {
state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth());
state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextMonth());
if (log.isTraceEnabled()) {
log.trace("[{}][{}] Updating state cycles (currentCycleTs={},nextCycleTs={})", state.getTenantId(), state.getEntityId(), state.getCurrentCycleTs(), state.getNextCycleTs());
}

8
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java

@ -19,11 +19,9 @@ import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.ConcurrentReferenceHashMap;
import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldLink;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
@ -31,8 +29,6 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg;
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.queue.util.AfterStartUp;
@ -56,8 +52,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache {
private final CalculatedFieldService calculatedFieldService;
private final TbelInvokeService tbelInvokeService;
private final ApiLimitService apiLimitService;
@Lazy
private final ActorSystemContext actorSystemContext;
private final ConcurrentMap<CalculatedFieldId, CalculatedField> calculatedFields = new ConcurrentHashMap<>();
private final ConcurrentMap<EntityId, List<CalculatedField>> entityIdCalculatedFields = new ConcurrentHashMap<>();
@ -75,7 +69,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache {
cfs.forEach(cf -> {
if (cf != null) {
calculatedFields.putIfAbsent(cf.getId(), cf);
actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf));
}
});
calculatedFields.values().forEach(cf -> {
@ -84,7 +77,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache {
PageDataIterable<CalculatedFieldLink> cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize);
cfls.forEach(link -> {
calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link);
actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link));
});
calculatedFieldLinks.values().stream()
.flatMap(List::stream)

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -110,6 +110,7 @@ public class CalculatedFieldCtx {
this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService);
initialized = true;
} catch (Exception e) {
initialized = false;
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e);
}
} else {
@ -123,6 +124,7 @@ public class CalculatedFieldCtx {
);
initialized = true;
} else {
initialized = false;
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.");
}
}

30
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java

@ -93,14 +93,13 @@ import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAS
@TbCoreComponent
public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase implements EdgeRpcService {
private static final int DESTROY_SESSION_MAX_ATTEMPTS = 10;
private final ConcurrentMap<EdgeId, EdgeGrpcSession> sessions = new ConcurrentHashMap<>();
private final ConcurrentMap<EdgeId, Lock> sessionNewEventsLocks = new ConcurrentHashMap<>();
private final Map<EdgeId, Boolean> sessionNewEvents = new HashMap<>();
private final ConcurrentMap<EdgeId, ScheduledFuture<?>> sessionEdgeEventChecks = new ConcurrentHashMap<>();
private final ConcurrentMap<UUID, Consumer<FromEdgeSyncResponse>> localSyncEdgeRequests = new ConcurrentHashMap<>();
private final ConcurrentMap<EdgeId, Boolean> edgeEventsMigrationProcessed = new ConcurrentHashMap<>();
private final List<EdgeGrpcSession> zombieSessions = new ArrayList<>();
@Value("${edges.rpc.port}")
private int rpcPort;
@ -193,7 +192,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
this.edgeEventProcessingExecutorService = ThingsBoardExecutors.newScheduledThreadPool(schedulerPoolSize, "edge-event-check-scheduler");
this.sendDownlinkExecutorService = ThingsBoardExecutors.newScheduledThreadPool(sendSchedulerPoolSize, "edge-send-scheduler");
this.executorService = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edge-service");
this.executorService.scheduleAtFixedRate(this::destroyKafkaSessionIfDisconnectedAndConsumerActive, 60, 60, TimeUnit.SECONDS);
this.executorService.scheduleAtFixedRate(this::cleanupZombieSessions, 60, 60, TimeUnit.SECONDS);
log.info("Edge RPC service initialized!");
}
@ -518,14 +517,10 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
private void destroySession(EdgeGrpcSession session) {
try (session) {
for (int i = 0; i < DESTROY_SESSION_MAX_ATTEMPTS; i++) {
if (session.destroy()) {
break;
} else {
try {
Thread.sleep(100);
} catch (InterruptedException ignored) {}
}
if (!session.destroy()) {
log.warn("[{}][{}] Session destroy failed for edge [{}] with session id [{}]. Adding to zombie queue for later cleanup.",
session.getTenantId(), session.getEdge().getId(), session.getEdge().getName(), session.getSessionId());
zombieSessions.add(session);
}
}
}
@ -634,7 +629,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
}
}
private void destroyKafkaSessionIfDisconnectedAndConsumerActive() {
private void cleanupZombieSessions() {
try {
List<EdgeId> toRemove = new ArrayList<>();
for (EdgeGrpcSession session : sessions.values()) {
@ -655,6 +650,17 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
}
}
}
zombieSessions.removeIf(zombie -> {
if (zombie.destroy()) {
log.info("[{}][{}] Successfully cleaned up zombie session [{}] for edge [{}].",
zombie.getTenantId(), zombie.getEdge().getId(), zombie.getSessionId(), zombie.getEdge().getName());
return true;
} else {
log.warn("[{}][{}] Failed to remove zombie session [{}] for edge [{}].",
zombie.getTenantId(), zombie.getEdge().getId(), zombie.getSessionId(), zombie.getEdge().getName());
return false;
}
});
} catch (Exception e) {
log.warn("Failed to cleanup kafka sessions", e);
}

2
application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java

@ -32,7 +32,7 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti
// This list should include all versions which are compatible for the upgrade.
// The compatibility cycle usually breaks when we have some scripts written in Java that may not work after new release.
private static final List<String> SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("4.1.0");
private static final List<String> SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("4.2.0");
private final ProjectInfo projectInfo;
private final JdbcTemplate jdbcTemplate;

2
application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java

@ -238,7 +238,7 @@ public class DefaultEntityQueryService implements EntityQueryService {
entitiesSortOrder = sortOrder;
}
EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, entitiesSortOrder);
return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, query.getKeyFilters());
return new EntityDataQuery(query.getEntityFilter(), edpl, query.getEntityFields(), query.getLatestValues(), query.getKeyFilters());
}
@Override

4
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java

@ -35,6 +35,7 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto;
@ -83,6 +84,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa
ActorSystemContext actorContext,
TbDeviceProfileCache deviceProfileCache,
TbAssetProfileCache assetProfileCache,
TbResourceDataCache tbResourceDataCache,
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService,
PartitionService partitionService,
@ -90,7 +92,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa
JwtSettingsService jwtSettingsService,
CalculatedFieldCache calculatedFieldCache,
CalculatedFieldStateService stateService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService,
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, tbResourceDataCache, calculatedFieldCache, apiUsageStateService, partitionService,
eventPublisher, jwtSettingsService);
this.queueFactory = tbQueueFactory;
this.stateService = stateService;

9
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.page.PageData;
@ -435,8 +436,9 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void onResourceChange(TbResourceInfo resource, TbQueueCallback callback) {
TenantId tenantId = resource.getTenantId();
TbResourceId resourceId = resource.getId();
if (resource.getResourceType() == ResourceType.LWM2M_MODEL) {
TenantId tenantId = resource.getTenantId();
log.trace("[{}][{}][{}] Processing change resource", tenantId, resource.getResourceType(), resource.getResourceKey());
ResourceUpdateMsg resourceUpdateMsg = ResourceUpdateMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
@ -447,6 +449,7 @@ public class DefaultTbClusterService implements TbClusterService {
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setResourceUpdateMsg(resourceUpdateMsg).build();
broadcast(transportMsg, DataConstants.LWM2M_TRANSPORT_NAME, callback);
}
broadcastEntityStateChangeEvent(tenantId, resourceId, ComponentLifecycleEvent.UPDATED);
}
@Override
@ -462,6 +465,7 @@ public class DefaultTbClusterService implements TbClusterService {
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setResourceDeleteMsg(resourceDeleteMsg).build();
broadcast(transportMsg, DataConstants.LWM2M_TRANSPORT_NAME, callback);
}
broadcastEntityStateChangeEvent(resource.getTenantId(), resource.getId(), ComponentLifecycleEvent.DELETED);
}
private <T> void broadcastEntityChangeToTransport(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) {
@ -592,7 +596,8 @@ public class DefaultTbClusterService implements TbClusterService {
EntityType.TENANT_PROFILE,
EntityType.DEVICE_PROFILE,
EntityType.ASSET_PROFILE,
EntityType.JOB)
EntityType.JOB,
EntityType.TB_RESOURCE)
|| (entityType == EntityType.ASSET && msg.getEvent() == ComponentLifecycleEvent.UPDATED)
|| (entityType == EntityType.DEVICE && msg.getEvent() == ComponentLifecycleEvent.UPDATED)
) {

4
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -55,6 +55,7 @@ import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.util.KvProtoUtil;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.dao.resource.ImageCacheKey;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
@ -176,10 +177,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
NotificationSchedulerService notificationSchedulerService,
NotificationRuleProcessor notificationRuleProcessor,
TbImageService imageService,
TbResourceDataCache tbResourceDataCache,
RuleEngineCallService ruleEngineCallService,
CalculatedFieldCache calculatedFieldCache,
EdqsService edqsService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService,
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, tbResourceDataCache, calculatedFieldCache, apiUsageStateService, partitionService,
eventPublisher, jwtSettingsService);
this.stateService = stateService;
this.localSubscriptionService = localSubscriptionService;

2
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java

@ -87,7 +87,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext,
StatsFactory statsFactory, EdgeContextComponent edgeCtx) {
super(actorContext, null, null, null, null, null, null,
super(actorContext, null, null, null, null, null, null, null,
null, null);
this.edgeCtx = edgeCtx;
this.stats = new EdgeConsumerStats(statsFactory);

4
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -33,6 +33,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg;
@ -78,13 +79,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedCo
QueueService queueService,
TbDeviceProfileCache deviceProfileCache,
TbAssetProfileCache assetProfileCache,
TbResourceDataCache tbResourceDataCache,
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService,
PartitionService partitionService,
ApplicationEventPublisher eventPublisher,
JwtSettingsService jwtSettingsService,
CalculatedFieldCache calculatedFieldCache) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, tbResourceDataCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
this.ctx = ctx;
this.tbDeviceRpcService = tbDeviceRpcService;
this.queueService = queueService;

5
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -30,12 +30,14 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -72,6 +74,7 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected final TbTenantProfileCache tenantProfileCache;
protected final TbDeviceProfileCache deviceProfileCache;
protected final TbAssetProfileCache assetProfileCache;
protected final TbResourceDataCache tbResourceDataCache;
protected final CalculatedFieldCache calculatedFieldCache;
protected final TbApiUsageStateService apiUsageStateService;
protected final PartitionService partitionService;
@ -202,6 +205,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
} else {
calculatedFieldCache.evict((CalculatedFieldId) componentLifecycleMsg.getEntityId());
}
} else if (EntityType.TB_RESOURCE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
tbResourceDataCache.evictResourceData(tenantId, new TbResourceId(componentLifecycleMsg.getEntityId().getId()));
}
eventPublisher.publishEvent(componentLifecycleMsg);

4
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.queue.processing;
import jakarta.annotation.PostConstruct;
import org.springframework.context.ApplicationEventPublisher;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
@ -43,12 +44,13 @@ public abstract class AbstractPartitionBasedConsumerService<N extends com.google
TbTenantProfileCache tenantProfileCache,
TbDeviceProfileCache deviceProfileCache,
TbAssetProfileCache assetProfileCache,
TbResourceDataCache tbResourceDataCache,
CalculatedFieldCache calculatedFieldCache,
TbApiUsageStateService apiUsageStateService,
PartitionService partitionService,
ApplicationEventPublisher eventPublisher,
JwtSettingsService jwtSettingsService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, tbResourceDataCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
}
@PostConstruct

1
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java

@ -102,7 +102,6 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
if (result.isSuccess()) {
logEntityActionService.logEntityAction(tenantId, resourceId, tbResource, actionType, user, resourceId.toString());
}
return result;
} catch (Exception e) {
logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.TB_RESOURCE),

10
application/src/main/resources/thingsboard.yml

@ -208,7 +208,7 @@ ui:
# Help parameters
help:
# Base URL for UI help assets
base-url: "${UI_HELP_BASE_URL:https://raw.githubusercontent.com/thingsboard/thingsboard-ui-help/release-4.2}"
base-url: "${UI_HELP_BASE_URL:https://raw.githubusercontent.com/thingsboard/thingsboard-ui-help/release-4.2.1 }"
# Database telemetry parameters
database:
@ -303,8 +303,11 @@ cassandra:
default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}"
# Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS, INDEFINITE
ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
# Enable/Disable timestamp key-value partioning on read queries
# Enable/Disable timestamp key-value partitioning on read queries
use_ts_key_value_partitioning_on_read: "${USE_TS_KV_PARTITIONING_ON_READ:true}"
# Safety trigger to fall back to use_ts_key_value_partitioning_on_read as true if estimated partitions count is greater than safety trigger value.
# It helps to prevent building huge partition list (OOM) for corner cases (like from 0 to infinity) and prefer fewer reads strategy from NoSQL database
use_ts_key_value_partitioning_on_read_max_estimated_partition_count: "${USE_TS_KV_PARTITIONING_ON_READ_MAX_ESTIMATED_PARTITION_COUNT:40}"
# The number of partitions that are cached in memory of each service. It is useful to decrease the load of re-inserting the same partitions again
ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}"
# Timeseries Time To Live (in seconds) for Cassandra Record. 0 - record has never expired
@ -676,6 +679,9 @@ cache:
maxSize: "${CACHE_SPECS_IMAGE_ETAGS_MAX_SIZE:10000}" # 0 means the cache is disabled
systemImagesBrowserTtlInMinutes: "${CACHE_SPECS_IMAGE_SYSTEM_BROWSER_TTL:0}" # Browser cache TTL for system images in minutes. 0 means the cache is disabled
tenantImagesBrowserTtlInMinutes: "${CACHE_SPECS_IMAGE_TENANT_BROWSER_TTL:0}" # Browser cache TTL for tenant images in minutes. 0 means the cache is disabled
tbResourceData:
timeToLiveInMinutes: "${CACHE_SPECS_RESOURCE_DATA_TTL:10080}" # TB resource data cache TTL
maxSize: "${CACHE_SPECS_RESOURCE_DATA_MAX_SIZE:100000}" # 0 means the cache is disabled
# Spring data parameters
spring.data.redis.repositories.enabled: false # Disable this because it is not required.

53
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java

@ -606,6 +606,59 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
});
}
@Test
public void testSimpleCalculatedFieldWhenCtxBecameUninitialized() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(testDevice.getId());
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName("M + 1");
calculatedField.setDebugSettings(DebugSettings.all());
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
Argument argument = new Argument();
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("m", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("m", argument));
config.setExpression("m + 1");
Output output = new Output();
output.setName("m1");
output.setType(OutputType.TIME_SERIES);
output.setDecimalsByDefault(0);
config.setOutput(output);
calculatedField.setConfiguration(config);
calculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"m\":1}"));
await().alias("create CF -> ctx is initialized -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode m1 = getLatestTelemetry(testDevice.getId(), "m1");
assertThat(m1).isNotNull();
assertThat(m1.get("m1").get(0).get("value").asText()).isEqualTo("2");
});
config.setExpression("m m");
calculatedField.setConfiguration(config);
calculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"m\":2}"));
await().alias("update CF -> ctx is not initialized -> no calculation performed").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode m1 = getLatestTelemetry(testDevice.getId(), "m1");
assertThat(m1).isNotNull();
assertThat(m1.get("m1").get(0).get("value").asText()).isEqualTo("2");
});
}
private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception {
return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class);
}

12
application/src/test/java/org/thingsboard/server/controller/AiModelControllerTest.java

@ -104,7 +104,10 @@ public class AiModelControllerTest extends AbstractControllerTest {
var model = doPost("/api/ai/model", constructValidOpenAiModel("Test model"), AiModel.class);
var newModelConfig = OpenAiChatModelConfig.builder()
.providerConfig(new OpenAiProviderConfig("test-api-key-updated"))
.providerConfig(OpenAiProviderConfig.builder()
.baseUrl(OpenAiProviderConfig.OPENAI_OFFICIAL_BASE_URL)
.apiKey("test-api-key-updated")
.build())
.modelId("o4-mini")
.temperature(0.2)
.topP(0.4)
@ -270,7 +273,7 @@ public class AiModelControllerTest extends AbstractControllerTest {
.tenantId(tenantId)
.name("Test model 1")
.configuration(OpenAiChatModelConfig.builder()
.providerConfig(new OpenAiProviderConfig("test-api-key"))
.providerConfig(OpenAiProviderConfig.builder().apiKey("test-api-key").build())
.modelId("o3-pro")
.build())
.build(), AiModel.class);
@ -594,7 +597,10 @@ public class AiModelControllerTest extends AbstractControllerTest {
private AiModel constructValidOpenAiModel(String name) {
var modelConfig = OpenAiChatModelConfig.builder()
.providerConfig(new OpenAiProviderConfig("test-api-key"))
.providerConfig(OpenAiProviderConfig.builder()
.baseUrl(OpenAiProviderConfig.OPENAI_OFFICIAL_BASE_URL)
.apiKey("test-api-key")
.build())
.modelId("gpt-4o")
.temperature(0.5)
.topP(0.3)

63
application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@ -519,6 +519,67 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
Assert.assertEquals(1, filteredAssetAlamData.getTotalElements());
}
@Test
public void testFindAlarmsWithEntityFilterAndLatestValues() throws Exception {
loginTenantAdmin();
List<Device> devices = new ArrayList<>();
List<String> temps = new ArrayList<>();
List<String> deviceNames = new ArrayList<>();
for (int i = 0; i < 10; i++) {
Device device = new Device();
device.setCustomerId(customerId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
device = doPost("/api/device", device, Device.class);
devices.add(device);
deviceNames.add(device.getName());
int temp = i * 10;
temps.add(String.valueOf(temp));
JsonNode content = JacksonUtil.toJsonNode("{\"temperature\": " + temp + "}");
doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + device.getUuidId() + "/timeseries/SERVER_SCOPE", content)
.andExpect(status().isOk());
Thread.sleep(1);
}
for (int i = 0; i < devices.size(); i++) {
Alarm alarm = new Alarm();
alarm.setCustomerId(customerId);
alarm.setOriginator(devices.get(i).getId());
String type = "device alarm" + i;
alarm.setType(type);
alarm.setSeverity(AlarmSeverity.WARNING);
doPost("/api/alarm", alarm, Alarm.class);
Thread.sleep(1);
}
AlarmDataPageLink pageLink = new AlarmDataPageLink();
pageLink.setPage(0);
pageLink.setPageSize(100);
pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.ALARM_FIELD, "created_time")));
List<EntityKey> alarmFields = new ArrayList<>();
alarmFields.add(new EntityKey(EntityKeyType.ALARM_FIELD, "type"));
List<EntityKey> entityFields = new ArrayList<>();
entityFields.add(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
List<EntityKey> latestValues = new ArrayList<>();
latestValues.add(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
EntityTypeFilter deviceTypeFilter = new EntityTypeFilter();
deviceTypeFilter.setEntityType(EntityType.DEVICE);
AlarmDataQuery deviceAlarmQuery = new AlarmDataQuery(deviceTypeFilter, pageLink, entityFields, latestValues, null, alarmFields);
PageData<AlarmData> alarmPageData = findAlarmsByQueryAndCheck(deviceAlarmQuery, 10);
List<String> retrievedAlarmTemps = alarmPageData.getData().stream().map(alarmData -> alarmData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).toList();
assertThat(retrievedAlarmTemps).containsExactlyInAnyOrderElementsOf(temps);
List<String> retrievedDeviceNames = alarmPageData.getData().stream().map(alarmData -> alarmData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).toList();
assertThat(retrievedDeviceNames).containsExactlyInAnyOrderElementsOf(deviceNames);
}
private void testCountAlarmsByQuery(List<Alarm> alarms) throws Exception {
AlarmCountQuery countQuery = new AlarmCountQuery();
@ -953,7 +1014,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
});
EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter);
EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter, keyFilters);
countByQueryAndCheck(countQuery, 97);
}

23
application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

@ -31,6 +31,7 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.DashboardInfo;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.StringUtils;
@ -321,18 +322,15 @@ public class TbResourceControllerTest extends AbstractControllerTest {
var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references");
Assert.assertNotNull(referenceValues);
var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<WidgetTypeInfo>>>() {
var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<EntityInfo>>>() {
});
Assert.assertNotNull(widgetTypeInfos);
Assert.assertFalse(widgetTypeInfos.isEmpty());
Assert.assertEquals(1, widgetTypeInfos.size());
var dashboardInfo = widgetTypeInfos.get(EntityType.WIDGET_TYPE.name()).get(0);
Assert.assertNotNull(dashboardInfo);
WidgetTypeInfo foundedWidgetType = doGet("/api/widgetTypeInfo/" + savedWidgetType.getId().getId().toString(), WidgetTypeInfo.class);
Assert.assertNotNull(foundedWidgetType);
Assert.assertEquals(foundedWidgetType, dashboardInfo);
var widgetTypeInfo = widgetTypeInfos.get(EntityType.WIDGET_TYPE.name()).get(0);
Assert.assertNotNull(widgetTypeInfo);
Assert.assertEquals(new EntityInfo(savedWidgetType.getId(), savedWidgetType.getName()), widgetTypeInfo);
}
@Test
@ -372,7 +370,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
Assert.assertTrue(isSuccess);
var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references");
var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<WidgetTypeInfo>>>() {
var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<EntityInfo>>>() {
});
Assert.assertNull(widgetTypeInfos);
}
@ -417,7 +415,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references");
Assert.assertNotNull(referenceValues);
var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<DashboardInfo>>>() {
var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<EntityInfo>>>() {
});
Assert.assertNotNull(dashboardInfos);
Assert.assertFalse(dashboardInfos.isEmpty());
@ -425,10 +423,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
var dashboardInfo = dashboardInfos.get(EntityType.DASHBOARD.name()).get(0);
Assert.assertNotNull(dashboardInfo);
DashboardInfo foundDashboard = doGet("/api/dashboard/info/" + savedDashboard.getId().getId().toString(), DashboardInfo.class);
Assert.assertNotNull(foundDashboard);
Assert.assertEquals(foundDashboard, dashboardInfo);
Assert.assertEquals(new EntityInfo(savedDashboard.getId(), savedDashboard.getName()), dashboardInfo);
}
@Test
@ -469,7 +464,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
Assert.assertTrue(isSuccess);
var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references");
var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<DashboardInfo>>>() {
var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference<HashMap<String, List<EntityInfo>>>() {
});
Assert.assertNull(dashboardInfos);
}

50
application/src/test/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateServiceTest.java

@ -20,9 +20,12 @@ import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.ApiUsageStateValue;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
@ -33,8 +36,17 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import java.lang.reflect.Field;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static java.time.ZoneOffset.UTC;
import static java.time.temporal.ChronoField.DAY_OF_MONTH;
import static java.time.temporal.ChronoUnit.MONTHS;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
@DaoSqlTest
@ -48,6 +60,7 @@ public class DefaultTbApiUsageStateServiceTest extends AbstractControllerTest {
private TenantId tenantId;
private Tenant savedTenant;
private TenantProfile savedTenantProfile;
private static final int MAX_ENABLE_VALUE = 5000;
private static final long VALUE_WARNING = 4500L;
@ -59,7 +72,7 @@ public class DefaultTbApiUsageStateServiceTest extends AbstractControllerTest {
loginSysAdmin();
TenantProfile tenantProfile = createTenantProfile();
TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class);
savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class);
Assert.assertNotNull(savedTenantProfile);
Tenant tenant = new Tenant();
@ -109,6 +122,41 @@ public class DefaultTbApiUsageStateServiceTest extends AbstractControllerTest {
assertEquals(ApiUsageStateValue.DISABLED, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState());
}
@Test
public void checkStartOfNextCycle_setsNextCycleToNextMonth() throws Exception {
ApiUsageState apiUsageState = new ApiUsageState(new ApiUsageStateId(UUID.randomUUID()));
apiUsageState.setDbStorageState(ApiUsageStateValue.ENABLED);
apiUsageState.setAlarmExecState(ApiUsageStateValue.ENABLED);
apiUsageState.setSmsExecState(ApiUsageStateValue.ENABLED);
apiUsageState.setTbelExecState(ApiUsageStateValue.ENABLED);
apiUsageState.setReExecState(ApiUsageStateValue.ENABLED);
apiUsageState.setTransportState(ApiUsageStateValue.ENABLED);
apiUsageState.setEmailExecState(ApiUsageStateValue.ENABLED);
apiUsageState.setJsExecState(ApiUsageStateValue.ENABLED);
apiUsageState.setTenantId(tenantId);
apiUsageState.setEntityId(tenantId);
long now = System.currentTimeMillis();
long currentCycleTs = now - TimeUnit.DAYS.toMillis(30);
long nextCycleTs = now - TimeUnit.MINUTES.toMillis(5); // < 1h ago
TenantApiUsageState tenantApiUsageState = new TenantApiUsageState(savedTenantProfile, apiUsageState);
tenantApiUsageState.setCycles(currentCycleTs, nextCycleTs);
Map<EntityId, BaseApiUsageState> map = new HashMap<>();
map.put(tenantId, tenantApiUsageState);
Field fieldToSet = DefaultTbApiUsageStateService.class.getDeclaredField("myUsageStates");
fieldToSet.setAccessible(true);
fieldToSet.set(service, map);
service.checkStartOfNextCycle();
long firstOfNextMonth = LocalDate.now()
.with((temporal) -> temporal.with(DAY_OF_MONTH, 1)
.plus(1, MONTHS))
.atStartOfDay(UTC).toInstant().toEpochMilli();
assertThat(tenantApiUsageState.getNextCycleTs()).isEqualTo(firstOfNextMonth);
}
private TenantProfile createTenantProfile() {
TenantProfile tenantProfile = new TenantProfile();
tenantProfile.setName("Tenant Profile");

2
application/src/test/java/org/thingsboard/server/service/entitiy/ai/DefaultTbAiModelServiceTest.java

@ -253,7 +253,7 @@ class DefaultTbAiModelServiceTest {
private static AiModelConfig constructValidOpenAiModelConfig() {
return OpenAiChatModelConfig.builder()
.providerConfig(new OpenAiProviderConfig("test-api-key"))
.providerConfig(OpenAiProviderConfig.builder().apiKey("test-api-key").build())
.modelId("gpt-4o")
.temperature(0.5)
.topP(0.3)

83
application/src/test/java/org/thingsboard/server/service/resource/DefaultResourceDataCacheTest.java

@ -0,0 +1,83 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.resource;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.bean.override.mockito.MockitoSpyBean;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.GeneralFileDescriptor;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.resource.TbResourceDataCache;
import org.thingsboard.server.dao.service.DaoSqlTest;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.clearInvocations;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
@DaoSqlTest
public class DefaultResourceDataCacheTest extends AbstractControllerTest {
@MockitoSpyBean
private ResourceService resourceService;
@Autowired
private TbResourceService tbResourceService;
@MockitoSpyBean
private TbResourceDataCache resourceDataCache;
@Test
public void testGetCachedResourceData() throws Exception {
loginTenantAdmin();
TbResource resource = new TbResource();
resource.setTenantId(tenantId);
resource.setTitle("File for AI request");
resource.setResourceType(ResourceType.GENERAL);
resource.setFileName("myTestJson.json");
GeneralFileDescriptor descriptor = new GeneralFileDescriptor("application/json");
resource.setDescriptorValue(descriptor);
byte[] data = "This is a test prompt for AI request.".getBytes();
resource.setData(data);
TbResourceInfo savedResource = tbResourceService.save(resource);
verify(resourceDataCache, timeout(2000).times(1)).evictResourceData(tenantId, savedResource.getId());
TbResourceDataInfo cachedData = resourceDataCache.getResourceDataInfoAsync(tenantId, savedResource.getId()).get();
assertThat(cachedData.getData()).isEqualTo(data);
assertThat(JacksonUtil.treeToValue(cachedData.getDescriptor(), GeneralFileDescriptor.class)).isEqualTo(descriptor);
verify(resourceService).getResourceDataInfo(tenantId, savedResource.getId());
// retrieve resource data second time
clearInvocations(resourceService);
TbResourceDataInfo cachedData2 = resourceDataCache.getResourceDataInfoAsync(tenantId, savedResource.getId()).get();
assertThat(cachedData2.getData()).isEqualTo(data);
verifyNoMoreInteractions(resourceService);
// delete resource, check cache
TbResource resourceById = resourceService.findResourceById(tenantId, savedResource.getId());
tbResourceService.delete(resourceById, true, null);
verify(resourceDataCache, timeout(2000).times(2)).evictResourceData(tenantId, savedResource.getId());
TbResourceDataInfo cachedDataAfterDeletion = resourceDataCache.getResourceDataInfoAsync(tenantId, savedResource.getId()).get();
assertThat(cachedDataAfterDeletion).isEqualTo(null);
}
}

125
application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.resource.sql;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -24,8 +25,10 @@ import org.junit.Test;
import org.junit.jupiter.api.Assertions;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.ai.TbAiNode;
import org.thingsboard.rule.engine.ai.TbAiNodeConfiguration;
import org.thingsboard.rule.engine.ai.TbResponseFormat;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.DashboardInfo;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.EntityType;
@ -37,26 +40,40 @@ import org.thingsboard.server.common.data.TbResourceInfoFilter;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.ai.AiModel;
import org.thingsboard.server.common.data.ai.model.chat.OpenAiChatModelConfig;
import org.thingsboard.server.common.data.ai.provider.OpenAiProviderConfig;
import org.thingsboard.server.common.data.debug.DebugSettings;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetTypeInfo;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.ai.AiModelService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.service.resource.TbResourceService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@ -135,6 +152,10 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
private WidgetTypeService widgetTypeService;
@Autowired
private DashboardService dashboardService;
@Autowired
private RuleChainService ruleChainService;
@Autowired
private AiModelService aiModelService;
private Tenant savedTenant;
private User tenantAdmin;
@ -453,11 +474,9 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
Assert.assertFalse(result.getReferences().isEmpty());
Assert.assertEquals(1, result.getReferences().size());
WidgetTypeInfo widgetTypeInfo = (WidgetTypeInfo) result.getReferences().get(EntityType.WIDGET_TYPE.name()).get(0);
WidgetTypeInfo foundWidgetTypeInfo = new WidgetTypeInfo(foundWidgetType);
EntityInfo widgetTypeInfo = (EntityInfo) result.getReferences().get(EntityType.WIDGET_TYPE.name()).get(0);
Assert.assertNotNull(widgetTypeInfo);
Assert.assertNotNull(foundWidgetTypeInfo);
Assert.assertEquals(widgetTypeInfo, foundWidgetTypeInfo);
Assert.assertEquals(widgetTypeInfo, new EntityInfo(foundWidgetType.getId(), foundWidgetType.getName()));
TbResourceInfo foundResourceInfo = resourceService.findResourceInfoById(savedTenant.getId(), savedResource.getId());
Assert.assertNotNull(foundResource);
@ -546,11 +565,9 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
Assert.assertNotNull(result.getReferences());
Assert.assertEquals(1, result.getReferences().size());
DashboardInfo dashboardInfo = (DashboardInfo) result.getReferences().get(EntityType.DASHBOARD.name()).get(0);
DashboardInfo foundDashboardInfo = dashboardService.findDashboardInfoById(savedTenant.getId(), savedDashboard.getId());
EntityInfo dashboardInfo = (EntityInfo) result.getReferences().get(EntityType.DASHBOARD.name()).get(0);
Assert.assertNotNull(dashboardInfo);
Assert.assertNotNull(foundDashboardInfo);
Assert.assertEquals(foundDashboardInfo, dashboardInfo);
Assert.assertEquals(new EntityInfo(savedDashboard.getId(), savedDashboard.getName()), dashboardInfo);
foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId());
Assert.assertNotNull(foundResource);
@ -598,6 +615,94 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
Assert.assertNull(foundResource);
}
@Test
public void testShouldNotDeleteResourceIfUsedInAiNode() throws Exception {
TbResource resource = new TbResource();
resource.setResourceType(ResourceType.GENERAL);
resource.setTitle("My resource");
resource.setFileName("test.json");
resource.setTenantId(savedTenant.getId());
resource.setData("<test></test>".getBytes());
TbResourceInfo savedResource = tbResourceService.save(resource);
RuleChainMetaData ruleChain = createRuleChainReferringResource(savedResource.getId());
TbResourceDeleteResult result = tbResourceService.delete(savedResource, false, null);
assertThat(result).isNotNull();
assertThat(result.isSuccess()).isFalse();
assertThat(result.getReferences()).isNotEmpty().hasSize(1);
EntityInfo entityInfo = (EntityInfo) result.getReferences().get(EntityType.RULE_CHAIN.name()).get(0);
assertThat(entityInfo).isEqualTo(new EntityInfo(ruleChain.getRuleChainId(), "Test"));
TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId());
assertThat(foundResource).isNotNull();
// force delete
TbResourceDeleteResult deleteResult = tbResourceService.delete(savedResource, true, null);
assertThat(deleteResult).isNotNull();
assertThat(deleteResult.isSuccess()).isTrue();
TbResource resourceAfterDeletion = resourceService.findResourceById(savedTenant.getId(), savedResource.getId());
assertThat(resourceAfterDeletion).isNull();
}
private RuleChainMetaData createRuleChainReferringResource(TbResourceId resourceId) {
AiModel model = constructValidOpenAiModel("Test model");
AiModel saved = aiModelService.save(model);
RuleChain ruleChain = new RuleChain();
ruleChain.setTenantId(tenantId);
ruleChain.setName("Test");
ruleChain.setType(RuleChainType.CORE);
ruleChain.setDebugMode(true);
ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b")));
ruleChain = ruleChainService.saveRuleChain(ruleChain);
RuleChainId ruleChainId = ruleChain.getId();
RuleChainMetaData metaData = new RuleChainMetaData();
metaData.setRuleChainId(ruleChainId);
RuleNode aiNode = new RuleNode();
aiNode.setName("Ai request");
aiNode.setType(org.thingsboard.rule.engine.ai.TbAiNode.class.getName());
aiNode.setConfigurationVersion(TbAiNode.class.getAnnotation(org.thingsboard.rule.engine.api.RuleNode.class).version());
aiNode.setDebugSettings(DebugSettings.all());
TbAiNodeConfiguration configuration = new TbAiNodeConfiguration();
configuration.setResourceIds(Set.of(resourceId.getId()));
configuration.setModelId(saved.getId());
configuration.setResponseFormat(new TbResponseFormat.TbJsonResponseFormat());
configuration.setTimeoutSeconds(1);
configuration.setUserPrompt("What is temp");
aiNode.setConfiguration(JacksonUtil.valueToTree(configuration));
metaData.setNodes(Arrays.asList(aiNode));
metaData.setFirstNodeIndex(0);
ruleChainService.saveRuleChainMetaData(tenantId, metaData, Function.identity());
return ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId);
}
private AiModel constructValidOpenAiModel(String name) {
var modelConfig = OpenAiChatModelConfig.builder()
.providerConfig(OpenAiProviderConfig.builder()
.baseUrl(OpenAiProviderConfig.OPENAI_OFFICIAL_BASE_URL)
.apiKey("test-api-key")
.build())
.modelId("gpt-4o")
.temperature(0.5)
.topP(0.3)
.frequencyPenalty(0.1)
.presencePenalty(0.2)
.maxOutputTokens(1000)
.timeoutSeconds(60)
.maxRetries(2)
.build();
return AiModel.builder()
.tenantId(tenantId)
.name(name)
.configuration(modelConfig)
.build();
}
@Test
public void testFindTenantResourcesByTenantId() throws Exception {
loginSysAdmin();

39
application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java

@ -25,10 +25,10 @@ import org.springframework.boot.test.mock.mockito.SpyBean;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.msg.gateway.metrics.GatewayMetadata;
import org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest;
import org.thingsboard.server.transport.mqtt.MqttTestConfigProperties;
import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService;
import org.thingsboard.server.common.msg.gateway.metrics.GatewayMetadata;
import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsState;
import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestCallback;
import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestClient;
@ -43,6 +43,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.any;
@ -112,6 +113,42 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt
processGatewayTelemetryTest(GATEWAY_TELEMETRY_TOPIC, expectedKeys, payload.getBytes(), deviceName1, deviceName2);
}
@Test
public void testAckIsReceivedOnFailedPublishMessage() throws Exception {
String devicePayload = "[{\"ts\": 10000, \"values\": " + PAYLOAD_VALUES_STR + "}]";
String payloadA = "{\"Device A\": " + devicePayload + "}";
String deviceBPayload = "[{\"ts\": 10000, \"values\": " + PAYLOAD_VALUES_STR + "}]";
String payloadB = "{\"Device B\": " + deviceBPayload + "}";
testAckIsReceivedOnFailedPublishMessage("Device A", payloadA.getBytes(), "Device B", payloadB.getBytes());
}
protected void testAckIsReceivedOnFailedPublishMessage(String deviceName1, byte[] payload1, String deviceName2, byte[] payload2) throws Exception {
updateDefaultTenantProfileConfig(profileConfiguration -> {
profileConfiguration.setMaxDevices(3);
});
MqttTestClient client = new MqttTestClient();
client.connectAndWait(gatewayAccessToken);
client.publishAndWait(GATEWAY_TELEMETRY_TOPIC, payload1);
// check device is created
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
assertNotNull(doGet("/api/tenant/devices?deviceName=" + deviceName1, Device.class));
});
client.publishAndWait(GATEWAY_TELEMETRY_TOPIC, payload2);
client.disconnectAndWait();
// check device was not created due to limit
doGet("/api/tenant/devices?deviceName=" + deviceName2).andExpect(status().isNotFound());
updateDefaultTenantProfileConfig(profileConfiguration -> {
profileConfiguration.setMaxDevices(0);
});
}
@Test
public void testGatewayConnect() throws Exception {
String payload = "{\"device\":\"Device A\"}";

11
application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesJsonIntegrationTest.java

@ -186,4 +186,15 @@ public abstract class AbstractMqttTimeseriesJsonIntegrationTest extends Abstract
assertFalse(callback.isPubAckReceived());
}
@Override
public void testAckIsReceivedOnFailedPublishMessage() throws Exception {
MqttTestConfigProperties configProperties = MqttTestConfigProperties.builder()
.deviceName("Test Post Telemetry device json payload")
.gatewayName("Test Post Telemetry gateway json payload")
.transportPayloadType(TransportPayloadType.JSON)
.telemetryTopicFilter(POST_DATA_TELEMETRY_TOPIC)
.build();
processBeforeTest(configProperties);
super.testAckIsReceivedOnFailedPublishMessage();
}
}

25
application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java

@ -459,6 +459,31 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac
assertFalse(callback.isPubAckReceived());
}
@Override
public void testAckIsReceivedOnFailedPublishMessage() throws Exception {
MqttTestConfigProperties configProperties = MqttTestConfigProperties.builder()
.deviceName("Test Post Telemetry device proto payload")
.gatewayName("Test Post Telemetry gateway proto payload")
.transportPayloadType(TransportPayloadType.PROTOBUF)
.telemetryTopicFilter(POST_DATA_TELEMETRY_TOPIC)
.build();
processBeforeTest(configProperties);
TransportApiProtos.GatewayTelemetryMsg.Builder gatewayTelemetryMsgProtoBuilder = TransportApiProtos.GatewayTelemetryMsg.newBuilder();
List<String> expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5");
String deviceName1 = "Device A";
String deviceName2 = "Device B";
TransportApiProtos.TelemetryMsg deviceATelemetryMsgProto = getDeviceTelemetryMsgProto(deviceName1, expectedKeys, 10000, 20000);
gatewayTelemetryMsgProtoBuilder.addAllMsg(List.of(deviceATelemetryMsgProto));
TransportApiProtos.GatewayTelemetryMsg payload1 = gatewayTelemetryMsgProtoBuilder.build();
TransportApiProtos.TelemetryMsg deviceBTelemetryMsgProto = getDeviceTelemetryMsgProto(deviceName2, expectedKeys, 10000, 20000);
TransportApiProtos.GatewayTelemetryMsg payload2 = TransportApiProtos.GatewayTelemetryMsg.newBuilder()
.addAllMsg(List.of(deviceBTelemetryMsgProto))
.build();
super.testAckIsReceivedOnFailedPublishMessage(deviceName1, payload1.toByteArray(), deviceName2, payload2.toByteArray());
}
private DynamicSchema getDynamicSchema() {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration);

5
common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java

@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.ResourceExportData;
import org.thingsboard.server.common.data.ResourceSubType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.common.data.TbResourceDeleteResult;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.TbResourceInfoFilter;
@ -46,6 +47,8 @@ public interface ResourceService extends EntityDaoService {
byte[] getResourceData(TenantId tenantId, TbResourceId resourceId);
TbResourceDataInfo getResourceDataInfo(TenantId tenantId, TbResourceId resourceId);
ResourceExportData exportResource(TbResourceInfo resourceInfo);
List<ResourceExportData> exportResources(TenantId tenantId, Collection<TbResourceInfo> resources);
@ -90,4 +93,6 @@ public interface ResourceService extends EntityDaoService {
TbResource createOrUpdateSystemResource(ResourceType resourceType, ResourceSubType resourceSubType, String resourceKey, byte[] data);
List<TbResourceInfo> findSystemOrTenantResourcesByIds(TenantId tenantId, List<TbResourceId> resourceIds);
}

20
common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java → common/dao-api/src/main/java/org/thingsboard/server/dao/resource/TbResourceDataCache.java

@ -13,22 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.cf;
package org.thingsboard.server.dao.resource;
import lombok.Data;
import org.thingsboard.server.common.data.cf.CalculatedField;
import com.google.common.util.concurrent.FluentFuture;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg;
@Data
public class CalculatedFieldInitMsg implements ToCalculatedFieldSystemMsg {
public interface TbResourceDataCache {
private final TenantId tenantId;
private final CalculatedField cf;
FluentFuture<TbResourceDataInfo> getResourceDataInfoAsync(TenantId tenantId, TbResourceId resourceId);
@Override
public MsgType getMsgType() {
return MsgType.CF_INIT_MSG;
}
void evictResourceData(TenantId tenantId, TbResourceId resourceId);
}

25
common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitProfileEntityMsg.java → common/data/src/main/java/org/thingsboard/server/common/data/GeneralFileDescriptor.java

@ -13,24 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.cf;
package org.thingsboard.server.common.data;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
@Data
public class CalculatedFieldInitProfileEntityMsg implements ToCalculatedFieldSystemMsg {
private final TenantId tenantId;
private final EntityId profileEntityId;
private final EntityId entityId;
@Override
public MsgType getMsgType() {
return MsgType.CF_INIT_PROFILE_ENTITY_MSG;
}
@EqualsAndHashCode
@AllArgsConstructor
@NoArgsConstructor
public class GeneralFileDescriptor {
private String mediaType;
}

3
common/data/src/main/java/org/thingsboard/server/common/data/ResourceType.java

@ -25,7 +25,8 @@ public enum ResourceType {
PKCS_12("application/x-pkcs12", false, false),
JS_MODULE("application/javascript", true, true),
IMAGE(null, true, true),
DASHBOARD("application/json", true, true);
DASHBOARD("application/json", true, true),
GENERAL(null, false, true);
@Getter
private final String mediaType;

7
common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java

@ -275,4 +275,11 @@ public class StringUtils {
return result;
}
public static String escapeControlChars(String text) {
return text
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
}

6
common/data/src/main/java/org/thingsboard/server/common/data/TbResource.java

@ -16,6 +16,7 @@
package org.thingsboard.server.common.data;
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonSetter;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@ -86,6 +87,11 @@ public class TbResource extends TbResourceInfo {
.orElse(null);
}
@JsonIgnore
public TbResourceDataInfo toResourceDataInfo() {
return new TbResourceDataInfo(data, getDescriptor());
}
@Override
public String toString() {
return super.toString();

21
common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java → common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDataInfo.java

@ -13,22 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.cf;
package org.thingsboard.server.common.data;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.thingsboard.server.common.data.cf.CalculatedFieldLink;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg;
import lombok.NoArgsConstructor;
@Data
public class CalculatedFieldLinkInitMsg implements ToCalculatedFieldSystemMsg {
@AllArgsConstructor
@NoArgsConstructor
public class TbResourceDataInfo {
private final TenantId tenantId;
private final CalculatedFieldLink link;
private byte[] data;
private JsonNode descriptor;
@Override
public MsgType getMsgType() {
return MsgType.CF_LINK_INIT_MSG;
}
}

3
common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java

@ -17,7 +17,6 @@ package org.thingsboard.server.common.data;
import lombok.Builder;
import lombok.Data;
import org.thingsboard.server.common.data.id.HasId;
import java.util.List;
import java.util.Map;
@ -27,6 +26,6 @@ import java.util.Map;
public class TbResourceDeleteResult {
private boolean success;
private Map<String, List<? extends HasId<?>>> references;
private Map<String, List<EntityInfo>> references;
}

6
common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbChatResponse.java

@ -22,7 +22,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
property = "status",
include = JsonTypeInfo.As.PROPERTY,
include = JsonTypeInfo.As.EXISTING_PROPERTY,
visible = true
)
@JsonSubTypes({
@ -51,9 +51,7 @@ public sealed interface TbChatResponse permits TbChatResponse.Success, TbChatRes
}
record Failure(
@Schema(
description = "A string containing details about the failure"
)
@Schema(description = "A string containing details about the failure")
String errorDetails
) implements TbChatResponse {

8
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/AiModelConfig.java

@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.ai.model.chat.GitHubModelsChatModelCon
import org.thingsboard.server.common.data.ai.model.chat.GoogleAiGeminiChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.GoogleVertexAiGeminiChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.MistralAiChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.OllamaChatModelConfig;
import org.thingsboard.server.common.data.ai.model.chat.OpenAiChatModelConfig;
import org.thingsboard.server.common.data.ai.provider.AiProvider;
import org.thingsboard.server.common.data.ai.provider.AiProviderConfig;
@ -34,6 +35,7 @@ import org.thingsboard.server.common.data.ai.provider.GitHubModelsProviderConfig
import org.thingsboard.server.common.data.ai.provider.GoogleAiGeminiProviderConfig;
import org.thingsboard.server.common.data.ai.provider.GoogleVertexAiGeminiProviderConfig;
import org.thingsboard.server.common.data.ai.provider.MistralAiProviderConfig;
import org.thingsboard.server.common.data.ai.provider.OllamaProviderConfig;
import org.thingsboard.server.common.data.ai.provider.OpenAiProviderConfig;
@JsonTypeInfo(
@ -50,7 +52,8 @@ import org.thingsboard.server.common.data.ai.provider.OpenAiProviderConfig;
@JsonSubTypes.Type(value = MistralAiChatModelConfig.class, name = "MISTRAL_AI"),
@JsonSubTypes.Type(value = AnthropicChatModelConfig.class, name = "ANTHROPIC"),
@JsonSubTypes.Type(value = AmazonBedrockChatModelConfig.class, name = "AMAZON_BEDROCK"),
@JsonSubTypes.Type(value = GitHubModelsChatModelConfig.class, name = "GITHUB_MODELS")
@JsonSubTypes.Type(value = GitHubModelsChatModelConfig.class, name = "GITHUB_MODELS"),
@JsonSubTypes.Type(value = OllamaChatModelConfig.class, name = "OLLAMA")
})
public interface AiModelConfig {
@ -69,7 +72,8 @@ public interface AiModelConfig {
@JsonSubTypes.Type(value = MistralAiProviderConfig.class, name = "MISTRAL_AI"),
@JsonSubTypes.Type(value = AnthropicProviderConfig.class, name = "ANTHROPIC"),
@JsonSubTypes.Type(value = AmazonBedrockProviderConfig.class, name = "AMAZON_BEDROCK"),
@JsonSubTypes.Type(value = GitHubModelsProviderConfig.class, name = "GITHUB_MODELS")
@JsonSubTypes.Type(value = GitHubModelsProviderConfig.class, name = "GITHUB_MODELS"),
@JsonSubTypes.Type(value = OllamaProviderConfig.class, name = "OLLAMA")
})
AiProviderConfig providerConfig();

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AiChatModelConfig.java

@ -24,7 +24,7 @@ public sealed interface AiChatModelConfig<C extends AiChatModelConfig<C>> extend
permits
OpenAiChatModelConfig, AzureOpenAiChatModelConfig, GoogleAiGeminiChatModelConfig,
GoogleVertexAiGeminiChatModelConfig, MistralAiChatModelConfig, AnthropicChatModelConfig,
AmazonBedrockChatModelConfig, GitHubModelsChatModelConfig {
AmazonBedrockChatModelConfig, GitHubModelsChatModelConfig, OllamaChatModelConfig {
ChatModel configure(Langchain4jChatModelConfigurer configurer);

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AmazonBedrockChatModelConfig.java

@ -33,7 +33,7 @@ public record AmazonBedrockChatModelConfig(
@NotBlank String modelId,
@PositiveOrZero Double temperature,
@Positive @Max(1) Double topP,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<AmazonBedrockChatModelConfig> {

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AnthropicChatModelConfig.java

@ -34,7 +34,7 @@ public record AnthropicChatModelConfig(
@PositiveOrZero Double temperature,
@Positive @Max(1) Double topP,
@PositiveOrZero Integer topK,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<AnthropicChatModelConfig> {

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/AzureOpenAiChatModelConfig.java

@ -35,7 +35,7 @@ public record AzureOpenAiChatModelConfig(
@Positive @Max(1) Double topP,
Double frequencyPenalty,
Double presencePenalty,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<AzureOpenAiChatModelConfig> {

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/GitHubModelsChatModelConfig.java

@ -35,7 +35,7 @@ public record GitHubModelsChatModelConfig(
@Positive @Max(1) Double topP,
Double frequencyPenalty,
Double presencePenalty,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<GitHubModelsChatModelConfig> {

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/GoogleAiGeminiChatModelConfig.java

@ -36,7 +36,7 @@ public record GoogleAiGeminiChatModelConfig(
@PositiveOrZero Integer topK,
Double frequencyPenalty,
Double presencePenalty,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<GoogleAiGeminiChatModelConfig> {

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/GoogleVertexAiGeminiChatModelConfig.java

@ -36,7 +36,7 @@ public record GoogleVertexAiGeminiChatModelConfig(
@PositiveOrZero Integer topK,
Double frequencyPenalty,
Double presencePenalty,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<GoogleVertexAiGeminiChatModelConfig> {

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/Langchain4jChatModelConfigurer.java

@ -35,4 +35,6 @@ public interface Langchain4jChatModelConfigurer {
ChatModel configureChatModel(GitHubModelsChatModelConfig chatModelConfig);
ChatModel configureChatModel(OllamaChatModelConfig chatModelConfig);
}

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/MistralAiChatModelConfig.java

@ -35,7 +35,7 @@ public record MistralAiChatModelConfig(
@Positive @Max(1) Double topP,
Double frequencyPenalty,
Double presencePenalty,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<MistralAiChatModelConfig> {

58
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/OllamaChatModelConfig.java

@ -0,0 +1,58 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.ai.model.chat;
import dev.langchain4j.model.chat.ChatModel;
import jakarta.validation.Valid;
import jakarta.validation.constraints.Max;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Positive;
import jakarta.validation.constraints.PositiveOrZero;
import lombok.Builder;
import lombok.With;
import org.thingsboard.server.common.data.ai.provider.AiProvider;
import org.thingsboard.server.common.data.ai.provider.OllamaProviderConfig;
@Builder
public record OllamaChatModelConfig(
@NotNull @Valid OllamaProviderConfig providerConfig,
@NotBlank String modelId,
@PositiveOrZero Double temperature,
@Positive @Max(1) Double topP,
@PositiveOrZero Integer topK,
Integer contextLength,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<OllamaChatModelConfig> {
@Override
public AiProvider provider() {
return AiProvider.OLLAMA;
}
@Override
public ChatModel configure(Langchain4jChatModelConfigurer configurer) {
return configurer.configureChatModel(this);
}
@Override
public boolean supportsJsonMode() {
return true;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/model/chat/OpenAiChatModelConfig.java

@ -35,7 +35,7 @@ public record OpenAiChatModelConfig(
@Positive @Max(1) Double topP,
Double frequencyPenalty,
Double presencePenalty,
@Positive Integer maxOutputTokens,
Integer maxOutputTokens,
@With @Positive Integer timeoutSeconds,
@With @PositiveOrZero Integer maxRetries
) implements AiChatModelConfig<OpenAiChatModelConfig> {

3
common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/AiProvider.java

@ -24,6 +24,7 @@ public enum AiProvider {
MISTRAL_AI,
ANTHROPIC,
AMAZON_BEDROCK,
GITHUB_MODELS
GITHUB_MODELS,
OLLAMA
}

2
common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/AiProviderConfig.java

@ -19,4 +19,4 @@ public sealed interface AiProviderConfig
permits
OpenAiProviderConfig, AzureOpenAiProviderConfig, GoogleAiGeminiProviderConfig,
GoogleVertexAiGeminiProviderConfig, MistralAiProviderConfig, AnthropicProviderConfig,
AmazonBedrockProviderConfig, GitHubModelsProviderConfig {}
AmazonBedrockProviderConfig, GitHubModelsProviderConfig, OllamaProviderConfig {}

48
common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/OllamaProviderConfig.java

@ -0,0 +1,48 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.ai.provider;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
public record OllamaProviderConfig(
@NotNull String baseUrl,
@NotNull @Valid OllamaAuth auth
) implements AiProviderConfig {
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type"
)
@JsonSubTypes({
@JsonSubTypes.Type(value = OllamaAuth.None.class, name = "NONE"),
@JsonSubTypes.Type(value = OllamaAuth.Basic.class, name = "BASIC"),
@JsonSubTypes.Type(value = OllamaAuth.Token.class, name = "TOKEN")
})
public sealed interface OllamaAuth {
record None() implements OllamaAuth {}
record Basic(@NotNull String username, @NotNull String password) implements OllamaAuth {}
record Token(@NotNull String token) implements OllamaAuth {}
}
}

30
common/data/src/main/java/org/thingsboard/server/common/data/ai/provider/OpenAiProviderConfig.java

@ -15,8 +15,32 @@
*/
package org.thingsboard.server.common.data.ai.provider;
import jakarta.validation.constraints.NotNull;
import com.fasterxml.jackson.annotation.JsonIgnore;
import jakarta.validation.constraints.AssertTrue;
import lombok.Builder;
import org.apache.commons.lang3.StringUtils;
import java.util.Objects;
@Builder
public record OpenAiProviderConfig(
@NotNull String apiKey
) implements AiProviderConfig {}
String baseUrl,
String apiKey
) implements AiProviderConfig {
public static final String OPENAI_OFFICIAL_BASE_URL = "https://api.openai.com/v1";
public OpenAiProviderConfig {
baseUrl = Objects.requireNonNullElse(baseUrl, OPENAI_OFFICIAL_BASE_URL);
}
@JsonIgnore
@AssertTrue(message = "API key is required when using the official OpenAI API")
public boolean isValid() {
if (baseUrl.equals(OPENAI_OFFICIAL_BASE_URL)) {
return StringUtils.isNotBlank(apiKey);
}
return true;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java

@ -37,7 +37,7 @@ public class UserFields extends AbstractEntityFields {
@Override
public String getName() {
return super.getEmail();
return getEmail();
}
public UserFields(UUID id, long createdTime, UUID tenantId, UUID customerId,

3
common/data/src/main/java/org/thingsboard/server/common/data/mobile/bundle/MobileAppBundle.java

@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.id.MobileAppId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.mobile.layout.MobileLayoutConfig;
import org.thingsboard.server.common.data.validation.Length;
import org.thingsboard.server.common.data.validation.NoXss;
@EqualsAndHashCode(callSuper = true)
@Data
@ -40,9 +41,11 @@ public class MobileAppBundle extends BaseData<MobileAppBundleId> implements HasT
private TenantId tenantId;
@Schema(description = "Application bundle title. Cannot be empty", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank
@NoXss
@Length(fieldName = "title")
private String title;
@Schema(description = "Application bundle description.")
@NoXss
@Length(fieldName = "description")
private String description;
@Schema(description = "Android application id")

1
common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRule.java

@ -62,6 +62,7 @@ public class NotificationRule extends BaseData<NotificationRuleId> implements Ha
@Valid
private NotificationRuleRecipientsConfig recipientsConfig;
@Valid
private NotificationRuleConfig additionalConfig;
private NotificationRuleId externalId;

2
common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleConfig.java

@ -16,12 +16,14 @@
package org.thingsboard.server.common.data.notification.rule;
import lombok.Data;
import org.thingsboard.server.common.data.validation.NoXss;
import java.io.Serializable;
@Data
public class NotificationRuleConfig implements Serializable {
@NoXss
private String description;
}

3
common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java

@ -137,9 +137,6 @@ public enum MsgType {
CF_CACHE_INIT_MSG, // Sent to init caches for CF actor;
CF_INIT_PROFILE_ENTITY_MSG, // Sent to init profile entities cache;
CF_INIT_MSG, // Sent to init particular calculated field;
CF_LINK_INIT_MSG, // Sent to init particular calculated field;
CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state;
CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures;

12
common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java

@ -60,16 +60,4 @@ public class SchedulerUtils {
return LocalDate.now(UTC).with(TemporalAdjusters.firstDayOfNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static long getStartOfNextNextMonth() {
return getStartOfNextNextMonth(UTC);
}
public static long getStartOfNextNextMonth(ZoneId zoneId) {
return LocalDate.now(UTC).with(firstDayOfNextNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static TemporalAdjuster firstDayOfNextNextMonth() {
return (temporal) -> temporal.with(DAY_OF_MONTH, 1).plus(2, MONTHS);
}
}

246
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java

@ -80,6 +80,8 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
@ -405,18 +407,34 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
protected void onDeviceTelemetryJson(int msgId, ByteBuf payload) throws AdaptorException {
JsonElement json = JsonMqttAdaptor.validateJsonPayload(sessionId, payload);
validateJsonObject(json);
for (Map.Entry<String, JsonElement> deviceEntry : json.getAsJsonObject().entrySet()) {
if (!deviceEntry.getValue().isJsonArray()) {
log.warn("{}[{}]", CAN_T_PARSE_VALUE, json);
continue;
}
List<Map.Entry<String, JsonElement>> deviceEntries = json.getAsJsonObject().entrySet().stream()
.filter(entry -> {
final boolean isArray = entry.getValue().isJsonArray();
if (!isArray) {
log.warn("{} device='{}' value={}", CAN_T_PARSE_VALUE, entry.getKey(), entry.getValue());
}
return isArray;
})
.toList();
if (deviceEntries.isEmpty()) {
log.debug("[{}][{}][{}] Devices telemetry message is empty", gateway.getTenantId(), gateway.getDeviceId(), sessionId);
throw new IllegalArgumentException("[" + sessionId + "] Devices telemetry message is empty for [" + gateway.getDeviceId() + "]");
}
AtomicInteger remaining = new AtomicInteger(deviceEntries.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
for (Map.Entry<String, JsonElement> deviceEntry : deviceEntries) {
String deviceName = deviceEntry.getKey();
process(deviceName, deviceCtx -> processPostTelemetryMsg(deviceCtx, deviceEntry.getValue(), deviceName, msgId),
t -> failedToProcessLog(deviceName, TELEMETRY, t));
process(deviceName, deviceCtx -> processPostTelemetryMsg(deviceCtx, deviceEntry.getValue(), deviceName, msgId,
remaining, ackSent),
t -> processFailure(msgId, deviceName, TELEMETRY, ackSent, t));
}
}
private void processPostTelemetryMsg(T deviceCtx, JsonElement msg, String deviceName, int msgId) {
private void processPostTelemetryMsg(T deviceCtx, JsonElement msg, String deviceName, int msgId, AtomicInteger remaining, AtomicBoolean ackSent) {
try {
long systemTs = System.currentTimeMillis();
TbPair<TransportProtos.PostTelemetryMsg, List<GatewayMetadata>> gatewayPayloadPair = JsonConverter.convertToGatewayTelemetry(msg.getAsJsonArray(), systemTs);
@ -425,10 +443,10 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
if (!CollectionUtils.isEmpty(metadata)) {
gatewayMetricsService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), metadata, systemTs);
}
transportService.process(deviceCtx.getSessionInfo(), postTelemetryMsg, getPubAckCallback(channel, deviceName, msgId, postTelemetryMsg));
transportService.process(deviceCtx.getSessionInfo(), postTelemetryMsg, getAggregatePubAckCallback(channel, msgId, deviceName, postTelemetryMsg, remaining, ackSent));
} catch (Throwable e) {
log.warn("[{}][{}][{}] Failed to convert telemetry: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, msg, e);
ackOrClose(msgId);
ackOrClose(msgId, ackSent);
}
}
@ -441,23 +459,28 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
throw new IllegalArgumentException("[" + sessionId + "] Devices telemetry messages is empty for [" + gateway.getDeviceId() + "]");
}
AtomicInteger remaining = new AtomicInteger(deviceMsgList.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
deviceMsgList.forEach(telemetryMsg -> {
String deviceName = checkDeviceName(telemetryMsg.getDeviceName());
process(deviceName, deviceCtx -> processPostTelemetryMsg(deviceCtx, telemetryMsg.getMsg(), deviceName, msgId),
t -> failedToProcessLog(deviceName, TELEMETRY, t));
process(deviceName, deviceCtx -> processPostTelemetryMsg(deviceCtx, telemetryMsg.getMsg(), deviceName, msgId,
remaining, ackSent),
t -> processFailure(msgId, deviceName, TELEMETRY, ackSent, t));
});
} catch (RuntimeException | InvalidProtocolBufferException e) {
throw new AdaptorException(e);
}
}
protected void processPostTelemetryMsg(MqttDeviceAwareSessionContext deviceCtx, TransportProtos.PostTelemetryMsg msg, String deviceName, int msgId) {
protected void processPostTelemetryMsg(MqttDeviceAwareSessionContext deviceCtx, TransportProtos.PostTelemetryMsg msg, String deviceName, int msgId,
AtomicInteger remaining, AtomicBoolean ackSent) {
try {
TransportProtos.PostTelemetryMsg postTelemetryMsg = ProtoConverter.validatePostTelemetryMsg(msg.toByteArray());
transportService.process(deviceCtx.getSessionInfo(), postTelemetryMsg, getPubAckCallback(channel, deviceName, msgId, postTelemetryMsg));
transportService.process(deviceCtx.getSessionInfo(), postTelemetryMsg, getAggregatePubAckCallback(channel, msgId, deviceName, postTelemetryMsg, remaining, ackSent));
} catch (Throwable e) {
log.warn("[{}][{}][{}] Failed to convert telemetry: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, msg, e);
ackOrClose(msgId);
ackOrClose(msgId, ackSent);
}
}
@ -475,26 +498,42 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
private void onDeviceClaimJson(int msgId, ByteBuf payload) throws AdaptorException {
JsonElement json = JsonMqttAdaptor.validateJsonPayload(sessionId, payload);
validateJsonObject(json);
for (Map.Entry<String, JsonElement> deviceEntry : json.getAsJsonObject().entrySet()) {
if (!deviceEntry.getValue().isJsonObject()) {
log.warn("{}[{}]", CAN_T_PARSE_VALUE, json);
continue;
}
List<Map.Entry<String, JsonElement>> deviceEntries = json.getAsJsonObject().entrySet().stream()
.filter(entry -> {
boolean isJsonObject = entry.getValue().isJsonObject();
if (!isJsonObject) {
log.warn("{} device='{}' value={}", CAN_T_PARSE_VALUE, entry.getKey(), entry.getValue());
}
return isJsonObject;
})
.toList();
if (deviceEntries.isEmpty()) {
log.debug("[{}][{}][{}] Devices claim message is empty", gateway.getTenantId(), gateway.getDeviceId(), sessionId);
throw new IllegalArgumentException("[" + sessionId + "] Devices claim message is empty for [" + gateway.getDeviceId() + "]");
}
AtomicInteger remaining = new AtomicInteger(deviceEntries.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
for (Map.Entry<String, JsonElement> deviceEntry : deviceEntries) {
String deviceName = deviceEntry.getKey();
process(deviceName, deviceCtx -> processClaimDeviceMsg(deviceCtx, deviceEntry.getValue(), deviceName, msgId),
t -> failedToProcessLog(deviceName, CLAIMING, t));
process(deviceName, deviceCtx -> processClaimDeviceMsg(deviceCtx, deviceEntry.getValue(), deviceName, msgId,
remaining, ackSent),
t -> processFailure(msgId, deviceName, CLAIMING, ackSent, t));
}
}
private void processClaimDeviceMsg(MqttDeviceAwareSessionContext deviceCtx, JsonElement claimRequest, String deviceName, int msgId) {
private void processClaimDeviceMsg(MqttDeviceAwareSessionContext deviceCtx, JsonElement claimRequest, String deviceName, int msgId,
AtomicInteger remaining, AtomicBoolean ackSent) {
try {
DeviceId deviceId = deviceCtx.getDeviceId();
TransportProtos.ClaimDeviceMsg claimDeviceMsg = JsonConverter.convertToClaimDeviceProto(deviceId, claimRequest);
transportService.process(deviceCtx.getSessionInfo(), claimDeviceMsg, getPubAckCallback(channel, deviceName, msgId, claimDeviceMsg));
transportService.process(deviceCtx.getSessionInfo(), claimDeviceMsg, getAggregatePubAckCallback(channel, msgId, deviceName, claimDeviceMsg, remaining, ackSent));
} catch (Throwable e) {
log.warn("[{}][{}][{}] Failed to convert claim message: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, claimRequest, e);
ackOrClose(msgId);
ackOrClose(msgId, ackSent);
}
}
@ -507,49 +546,70 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
throw new IllegalArgumentException("[" + sessionId + "] Devices claim messages is empty for [" + gateway.getDeviceId() + "]");
}
AtomicInteger remaining = new AtomicInteger(claimMsgList.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
claimMsgList.forEach(claimDeviceMsg -> {
String deviceName = checkDeviceName(claimDeviceMsg.getDeviceName());
process(deviceName, deviceCtx -> processClaimDeviceMsg(deviceCtx, claimDeviceMsg.getClaimRequest(), deviceName, msgId),
t -> failedToProcessLog(deviceName, CLAIMING, t));
process(deviceName, deviceCtx -> processClaimDeviceMsg(deviceCtx, claimDeviceMsg.getClaimRequest(), deviceName, msgId,
remaining, ackSent),
t -> processFailure(msgId, deviceName, CLAIMING, ackSent, t));
});
} catch (RuntimeException | InvalidProtocolBufferException e) {
throw new AdaptorException(e);
}
}
private void processClaimDeviceMsg(MqttDeviceAwareSessionContext deviceCtx, TransportApiProtos.ClaimDevice claimRequest, String deviceName, int msgId) {
private void processClaimDeviceMsg(MqttDeviceAwareSessionContext deviceCtx, TransportApiProtos.ClaimDevice claimRequest, String deviceName, int msgId,
AtomicInteger remaining, AtomicBoolean ackSent) {
try {
DeviceId deviceId = deviceCtx.getDeviceId();
TransportProtos.ClaimDeviceMsg claimDeviceMsg = ProtoConverter.convertToClaimDeviceProto(deviceId, claimRequest.toByteArray());
transportService.process(deviceCtx.getSessionInfo(), claimDeviceMsg, getPubAckCallback(channel, deviceName, msgId, claimDeviceMsg));
transportService.process(deviceCtx.getSessionInfo(), claimDeviceMsg, getAggregatePubAckCallback(channel, msgId, deviceName, claimDeviceMsg, remaining, ackSent));
} catch (Throwable e) {
log.warn("[{}][{}][{}] Failed to convert claim message: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, claimRequest, e);
ackOrClose(msgId);
ackOrClose(msgId, ackSent);
}
}
private void onDeviceAttributesJson(int msgId, ByteBuf payload) throws AdaptorException {
JsonElement json = JsonMqttAdaptor.validateJsonPayload(sessionId, payload);
validateJsonObject(json);
for (Map.Entry<String, JsonElement> deviceEntry : json.getAsJsonObject().entrySet()) {
if (!deviceEntry.getValue().isJsonObject()) {
log.warn("{}[{}]", CAN_T_PARSE_VALUE, json);
continue;
}
List<Map.Entry<String, JsonElement>> deviceEntries = json.getAsJsonObject().entrySet().stream()
.filter(entry -> {
boolean isJsonObject = entry.getValue().isJsonObject();
if (!isJsonObject) {
log.warn("{} device='{}' value={}", CAN_T_PARSE_VALUE, entry.getKey(), entry.getValue());
}
return isJsonObject;
})
.toList();
if (deviceEntries.isEmpty()) {
log.debug("[{}][{}][{}] Devices attribute message is empty", gateway.getTenantId(), gateway.getDeviceId(), sessionId);
throw new IllegalArgumentException("[" + sessionId + "] Devices attribute message is empty for [" + gateway.getDeviceId() + "]");
}
AtomicInteger remaining = new AtomicInteger(deviceEntries.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
for (Map.Entry<String, JsonElement> deviceEntry : deviceEntries) {
String deviceName = deviceEntry.getKey();
process(deviceName, deviceCtx -> processPostAttributesMsg(deviceCtx, deviceEntry.getValue(), deviceName, msgId),
t -> failedToProcessLog(deviceName, ATTRIBUTE, t));
process(deviceName, deviceCtx -> processPostAttributesMsg(deviceCtx, deviceEntry.getValue(), deviceName, msgId,
remaining, ackSent),
t -> processFailure(msgId, deviceName, ATTRIBUTE, ackSent, t));
}
}
private void processPostAttributesMsg(MqttDeviceAwareSessionContext deviceCtx, JsonElement msg, String deviceName, int msgId) {
private void processPostAttributesMsg(MqttDeviceAwareSessionContext deviceCtx, JsonElement msg, String deviceName, int msgId,
AtomicInteger remaining, AtomicBoolean ackSent) {
try {
TransportProtos.PostAttributeMsg postAttributeMsg = JsonConverter.convertToAttributesProto(msg.getAsJsonObject());
transportService.process(deviceCtx.getSessionInfo(), postAttributeMsg, getPubAckCallback(channel, deviceName, msgId, postAttributeMsg));
transportService.process(deviceCtx.getSessionInfo(), postAttributeMsg, getAggregatePubAckCallback(channel, msgId, deviceName, postAttributeMsg, remaining, ackSent));
} catch (Throwable e) {
log.warn("[{}][{}][{}] Failed to process device attributes command: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, msg, e);
ackOrClose(msgId);
ackOrClose(msgId, ackSent);
}
}
@ -562,23 +622,28 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
throw new IllegalArgumentException("[" + sessionId + "] Devices attributes keys list is empty for [" + gateway.getDeviceId() + "]");
}
AtomicInteger remaining = new AtomicInteger(attributesMsgList.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
attributesMsgList.forEach(attributesMsg -> {
String deviceName = checkDeviceName(attributesMsg.getDeviceName());
process(deviceName, deviceCtx -> processPostAttributesMsg(deviceCtx, attributesMsg.getMsg(), deviceName, msgId),
t -> failedToProcessLog(deviceName, ATTRIBUTE, t));
process(deviceName, deviceCtx -> processPostAttributesMsg(deviceCtx, attributesMsg.getMsg(), deviceName, msgId,
remaining, ackSent),
t -> processFailure(msgId, deviceName, ATTRIBUTE, ackSent, t));
});
} catch (RuntimeException | InvalidProtocolBufferException e) {
throw new AdaptorException(e);
}
}
protected void processPostAttributesMsg(MqttDeviceAwareSessionContext deviceCtx, TransportProtos.PostAttributeMsg kvListProto, String deviceName, int msgId) {
protected void processPostAttributesMsg(MqttDeviceAwareSessionContext deviceCtx, TransportProtos.PostAttributeMsg kvListProto, String deviceName, int msgId,
AtomicInteger remaining, AtomicBoolean ackSent) {
try {
TransportProtos.PostAttributeMsg postAttributeMsg = ProtoConverter.validatePostAttributeMsg(kvListProto);
transportService.process(deviceCtx.getSessionInfo(), postAttributeMsg, getPubAckCallback(channel, deviceName, msgId, postAttributeMsg));
transportService.process(deviceCtx.getSessionInfo(), postAttributeMsg, getAggregatePubAckCallback(channel, msgId, deviceName, postAttributeMsg, remaining, ackSent));
} catch (Throwable e) {
log.warn("[{}][{}][{}] Failed to process device attributes command: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, kvListProto, e);
ackOrClose(msgId);
ackOrClose(msgId, ackSent);
}
}
@ -647,27 +712,34 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
}
private void onDeviceRpcResponse(Integer requestId, String data, String deviceName, int msgId) {
process(deviceName, deviceCtx -> processRpcResponseMsg(deviceCtx, requestId, data, deviceName, msgId),
t -> failedToProcessLog(deviceName, RPC_RESPONSE, t));
AtomicInteger remaining = new AtomicInteger(1);
AtomicBoolean ackSent = new AtomicBoolean(false);
process(deviceName, deviceCtx -> processRpcResponseMsg(deviceCtx, requestId, data, deviceName, msgId, remaining, ackSent),
t -> processFailure(msgId, deviceName, RPC_RESPONSE, ackSent, t));
}
private void processRpcResponseMsg(MqttDeviceAwareSessionContext deviceCtx, Integer requestId, String data, String deviceName, int msgId) {
private void processRpcResponseMsg(MqttDeviceAwareSessionContext deviceCtx, Integer requestId, String data, String deviceName,
int msgId, AtomicInteger remaining, AtomicBoolean ackSent) {
TransportProtos.ToDeviceRpcResponseMsg rpcResponseMsg = TransportProtos.ToDeviceRpcResponseMsg.newBuilder()
.setRequestId(requestId).setPayload(data).build();
transportService.process(deviceCtx.getSessionInfo(), rpcResponseMsg, getPubAckCallback(channel, deviceName, msgId, rpcResponseMsg));
transportService.process(deviceCtx.getSessionInfo(), rpcResponseMsg,
getAggregatePubAckCallback(channel, msgId, deviceName, rpcResponseMsg, remaining, ackSent));
}
private void processGetAttributeRequestMessage(MqttPublishMessage mqttMsg, String deviceName, TransportProtos.GetAttributeRequestMsg requestMsg) {
int msgId = getMsgId(mqttMsg);
process(deviceName, deviceCtx -> processGetAttributeRequestMessage(deviceCtx, requestMsg, deviceName, msgId),
t -> {
failedToProcessLog(deviceName, ATTRIBUTES_REQUEST, t);
ack(mqttMsg, MqttReasonCodes.PubAck.IMPLEMENTATION_SPECIFIC_ERROR);
});
AtomicInteger remaining = new AtomicInteger(1);
AtomicBoolean ackSent = new AtomicBoolean(false);
process(deviceName, deviceCtx -> {
processGetAttributeRequestMessage(deviceCtx, requestMsg, deviceName, msgId, remaining, ackSent);
},
t -> processFailure(msgId, deviceName, ATTRIBUTES_REQUEST, ackSent, MqttReasonCodes.PubAck.IMPLEMENTATION_SPECIFIC_ERROR, t));
}
private void processGetAttributeRequestMessage(T deviceCtx, TransportProtos.GetAttributeRequestMsg requestMsg, String deviceName, int msgId) {
transportService.process(deviceCtx.getSessionInfo(), requestMsg, getPubAckCallback(channel, deviceName, msgId, requestMsg));
private void processGetAttributeRequestMessage(T deviceCtx, TransportProtos.GetAttributeRequestMsg requestMsg,
String deviceName, int msgId, AtomicInteger remaining, AtomicBoolean ackSent) {
transportService.process(deviceCtx.getSessionInfo(), requestMsg,
getAggregatePubAckCallback(channel, msgId, deviceName, requestMsg, remaining, ackSent));
}
private TransportProtos.GetAttributeRequestMsg toGetAttributeRequestMsg(int requestId, boolean clientScope, Set<String> keys) {
@ -718,9 +790,11 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
}
}
protected void ackOrClose(int msgId) {
protected void ackOrClose(int msgId, AtomicBoolean ackSent) {
if (MqttVersion.MQTT_5.equals(deviceSessionCtx.getMqttVersion())) {
ack(msgId, MqttReasonCodes.PubAck.PAYLOAD_FORMAT_INVALID);
if (ackSent.compareAndSet(false, true)) {
ack(msgId, MqttReasonCodes.PubAck.PAYLOAD_FORMAT_INVALID);
}
} else {
channel.close();
}
@ -742,23 +816,38 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
keyValueProtoBuilder.setType(TransportProtos.KeyValueType.STRING_V);
keyValueProtoBuilder.setStringV(connectionState.name());
TransportProtos.PostTelemetryMsg postTelemetryMsg = postTelemetryMsgCreated(keyValueProtoBuilder.build(), ts);
transportService.process(sessionInfo, postTelemetryMsg, getPubAckCallback(channel, deviceName, -1, postTelemetryMsg));
TransportServiceCallback<Void> pubAckCallback = getAggregatePubAckCallback(channel, -1, deviceName, postTelemetryMsg,
new AtomicInteger(1), new AtomicBoolean(false));
transportService.process(sessionInfo, postTelemetryMsg, pubAckCallback);
}
public ConcurrentMap<String, T> getDevices () {
public ConcurrentMap<String, T> getDevices() {
return this.devices;
}
private <T> TransportServiceCallback<Void> getPubAckCallback(final ChannelHandlerContext ctx, final String deviceName, final int msgId, final T msg) {
protected <T>TransportServiceCallback<Void> getAggregatePubAckCallback(
final ChannelHandlerContext ctx,
final int msgId,
final String deviceName,
final T msg,
final AtomicInteger remaining,
final AtomicBoolean ackSent) {
return new TransportServiceCallback<Void>() {
@Override
public void onSuccess(Void dummy) {
log.trace("[{}][{}][{}][{}] Published msg: [{}]", gateway.getTenantId(), gateway.getDeviceId(), sessionId, deviceName, msg);
if (msgId > 0) {
ctx.writeAndFlush(MqttTransportHandler.createMqttPubAckMsg(deviceSessionCtx, msgId, MqttReasonCodes.PubAck.SUCCESS.byteValue()));
} else {
log.trace("[{}][{}][{}] Wrong msg id: [{}]", gateway.getTenantId(), gateway.getDeviceId(), sessionId, msg);
ctx.writeAndFlush(MqttTransportHandler.createMqttPubAckMsg(deviceSessionCtx, msgId, MqttReasonCodes.PubAck.UNSPECIFIED_ERROR.byteValue()));
if (remaining.decrementAndGet() == 0 && ackSent.compareAndSet(false, true)) {
if (msgId > 0) {
ctx.writeAndFlush(MqttTransportHandler.createMqttPubAckMsg(
deviceSessionCtx, msgId, MqttReasonCodes.PubAck.SUCCESS.byteValue()));
} else {
log.trace("[{}][{}][{}] Wrong msg id: [{}]", gateway.getTenantId(), gateway.getDeviceId(), sessionId, msgId);
ctx.writeAndFlush(MqttTransportHandler.createMqttPubAckMsg(
deviceSessionCtx, msgId, MqttReasonCodes.PubAck.UNSPECIFIED_ERROR.byteValue()));
}
}
if (msgId <= 0) {
closeDeviceSession(deviceName, MqttReasonCodes.Disconnect.MALFORMED_PACKET);
}
}
@ -767,11 +856,20 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
public void onError(Throwable e) {
log.trace("[{}][{}][{}] Failed to publish msg: [{}] for device: [{}]", gateway.getTenantId(), gateway.getDeviceId(), sessionId, msg, deviceName, e);
if (e instanceof TbRateLimitsException) {
if (ackSent.compareAndSet(false, true)) {
ctx.writeAndFlush(MqttTransportHandler.createMqttPubAckMsg(
deviceSessionCtx, msgId, MqttReasonCodes.PubAck.QUOTA_EXCEEDED.byteValue()));
ctx.close();
}
closeDeviceSession(deviceName, MqttReasonCodes.Disconnect.MESSAGE_RATE_TOO_HIGH);
} else {
if (ackSent.compareAndSet(false, true)) {
ctx.writeAndFlush(MqttTransportHandler.createMqttPubAckMsg(
deviceSessionCtx, msgId, MqttReasonCodes.PubAck.UNSPECIFIED_ERROR.byteValue()));
ctx.close();
}
closeDeviceSession(deviceName, MqttReasonCodes.Disconnect.UNSPECIFIED_ERROR);
}
ctx.close();
}
};
}
@ -790,10 +888,20 @@ public abstract class AbstractGatewaySessionHandler<T extends AbstractGatewayDev
}
}
protected void failedToProcessLog(String deviceName, String msgType, Throwable t) {
log.debug("[{}][{}][{}] Failed to process device {} command: [{}]", gateway.getTenantId(), gateway.getDeviceId(), sessionId, msgType, deviceName, t);
protected void processFailure(int msgId, String deviceName, String msgType, AtomicBoolean ackSent, Throwable t) {
if (DataConstants.MAXIMUM_NUMBER_OF_DEVICES_REACHED.equals(t.getMessage())) {
processFailure(msgId, deviceName, msgType, ackSent, MqttReasonCodes.PubAck.QUOTA_EXCEEDED, t);
} else {
processFailure(msgId, deviceName, msgType, ackSent, MqttReasonCodes.PubAck.UNSPECIFIED_ERROR, t);
}
}
protected void processFailure(int msgId, String deviceName, String msgType, AtomicBoolean ackSent, MqttReasonCodes.PubAck pubAck, Throwable t) {
log.debug("[{}][{}][{}] Failed to process device {} command: [{}]", gateway.getTenantId(), gateway.getDeviceId(), sessionId, msgType, deviceName, t);
if (ackSent.compareAndSet(false, true)) {
ack(msgId, pubAck);
}
}
private void closeDeviceSession(String deviceName, MqttReasonCodes.Disconnect returnCode) {
try {

27
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java

@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import com.google.gson.JsonSyntaxException;
import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.handler.codec.mqtt.MqttPublishMessage;
import io.netty.handler.codec.mqtt.MqttReasonCodes;
import io.netty.handler.codec.mqtt.MqttTopicSubscription;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@ -48,6 +49,8 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.transport.mqtt.util.sparkplug.SparkplugConnectionState.ONLINE;
import static org.thingsboard.server.transport.mqtt.util.sparkplug.SparkplugMessageType.DBIRTH;
@ -144,37 +147,49 @@ public class SparkplugNodeSessionHandler extends AbstractGatewaySessionHandler<S
public void onDeviceTelemetryProto(ListenableFuture<MqttDeviceAwareSessionContext> contextListenableFuture,
int msgId, List<TransportProtos.PostTelemetryMsg> postTelemetryMsgList, String deviceName) {
if (CollectionUtils.isEmpty(postTelemetryMsgList)) {
log.debug("[{}] Device telemetry list is empty for: [{}]", sessionId, gateway.getDeviceId());
}
AtomicInteger remaining = new AtomicInteger(postTelemetryMsgList.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
process(contextListenableFuture, deviceCtx -> {
for (TransportProtos.PostTelemetryMsg telemetryMsg : postTelemetryMsgList) {
try {
processPostTelemetryMsg(deviceCtx, telemetryMsg, deviceName, msgId);
processPostTelemetryMsg(deviceCtx, telemetryMsg, deviceName, msgId, remaining, ackSent);
} catch (Throwable e) {
log.warn("[{}][{}] Failed to convert telemetry: {}", gateway.getDeviceId(), deviceName, telemetryMsg, e);
ackOrClose(msgId);
ackOrClose(msgId, ackSent);
}
}
},
t -> log.debug("[{}] Failed to process device telemetry command: {}", sessionId, deviceName, t));
t -> processFailure(msgId, deviceName, "Failed to process device telemetry command", ackSent, t));
}
private void onDeviceAttributesProto(ListenableFuture<MqttDeviceAwareSessionContext> contextListenableFuture, int msgId,
List<TransportApiProtos.AttributesMsg> attributesMsgList, String deviceName) throws AdaptorException {
try {
if (CollectionUtils.isEmpty(attributesMsgList)) {
log.debug("[{}] Devices attributes keys list is empty for: [{}]", sessionId, gateway.getDeviceId());
log.debug("[{}] Device attribute list is empty for: [{}]", sessionId, gateway.getDeviceId());
}
AtomicInteger remaining = new AtomicInteger(attributesMsgList.size());
AtomicBoolean ackSent = new AtomicBoolean(false);
process(contextListenableFuture, deviceCtx -> {
for (TransportApiProtos.AttributesMsg attributesMsg : attributesMsgList) {
TransportProtos.PostAttributeMsg kvListProto = attributesMsg.getMsg();
try {
TransportProtos.PostAttributeMsg postAttributeMsg = ProtoConverter.validatePostAttributeMsg(kvListProto);
processPostAttributesMsg(deviceCtx, postAttributeMsg, deviceName, msgId);
processPostAttributesMsg(deviceCtx, postAttributeMsg, deviceName, msgId, remaining, ackSent);
} catch (Throwable e) {
log.warn("[{}][{}] Failed to process device attributes command: {}", gateway.getDeviceId(), deviceName, kvListProto, e);
ackOrClose(msgId, ackSent);
}
}
},
t -> log.debug("[{}] Failed to process device attributes command: {}", sessionId, deviceName, t));
t -> processFailure(msgId, deviceName, "Failed to process device attributes command", ackSent, t));
} catch (RuntimeException e) {
throw new AdaptorException(e);
}

15
common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java

@ -15,12 +15,15 @@
*/
package org.thingsboard.common.util;
import com.google.common.util.concurrent.FluentFuture;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.function.Consumer;
@ -71,4 +74,16 @@ public class DonAsynchron {
return future;
}
public static <T> FluentFuture<T> toFluentFuture(CompletableFuture<T> completable) {
SettableFuture<T> future = SettableFuture.create();
completable.whenComplete((result, exception) -> {
if (exception != null) {
future.setException(exception);
} else {
future.set(result);
}
});
return FluentFuture.from(future);
}
}

5
dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.dao;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
@ -22,8 +23,8 @@ import java.util.List;
public interface ResourceContainerDao<T extends HasId<?>> {
List<T> findByTenantIdAndResourceLink(TenantId tenantId, String link, int limit);
List<EntityInfo> findByTenantIdAndResource(TenantId tenantId, String reference, int limit);
List<T> findByResourceLink(String link, int limit);
List<EntityInfo> findByResource(String reference, int limit);
}

5
dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java

@ -50,6 +50,7 @@ import org.thingsboard.server.dao.ImageContainerDao;
import org.thingsboard.server.dao.asset.AssetProfileDao;
import org.thingsboard.server.dao.dashboard.DashboardInfoDao;
import org.thingsboard.server.dao.device.DeviceProfileDao;
import org.thingsboard.server.dao.rule.RuleChainDao;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.ResourceDataValidator;
import org.thingsboard.server.dao.util.ImageUtils;
@ -109,8 +110,8 @@ public class BaseImageService extends BaseResourceService implements ImageServic
public BaseImageService(TbResourceDao resourceDao, TbResourceInfoDao resourceInfoDao, ResourceDataValidator resourceValidator,
AssetProfileDao assetProfileDao, DeviceProfileDao deviceProfileDao, WidgetsBundleDao widgetsBundleDao,
WidgetTypeDao widgetTypeDao, DashboardInfoDao dashboardInfoDao) {
super(resourceDao, resourceInfoDao, resourceValidator, widgetTypeDao, dashboardInfoDao);
WidgetTypeDao widgetTypeDao, DashboardInfoDao dashboardInfoDao, RuleChainDao ruleChainDao) {
super(resourceDao, resourceInfoDao, resourceValidator, widgetTypeDao, dashboardInfoDao, ruleChainDao);
this.assetProfileDao = assetProfileDao;
this.deviceProfileDao = deviceProfileDao;
this.widgetsBundleDao = widgetsBundleDao;

72
dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java

@ -35,11 +35,13 @@ import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoEvictEvent;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceExportData;
import org.thingsboard.server.common.data.ResourceSubType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.common.data.TbResourceDeleteResult;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.TbResourceInfoFilter;
@ -56,6 +58,7 @@ import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.rule.RuleChainDao;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.ResourceDataValidator;
@ -92,13 +95,16 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
protected final ResourceDataValidator resourceValidator;
protected final WidgetTypeDao widgetTypeDao;
protected final DashboardInfoDao dashboardInfoDao;
private final Map<EntityType, ResourceContainerDao<?>> resourceContainerDaoMap = new HashMap<>();
protected final RuleChainDao ruleChainDao;
private final Map<EntityType, ResourceContainerDao<?>> resourceLinkContainerDaoMap = new HashMap<>();
private final Map<EntityType, ResourceContainerDao<?>> generalResourceContainerDaoMap = new HashMap<>();
protected static final int MAX_ENTITIES_TO_FIND = 10;
@PostConstruct
public void init() {
resourceContainerDaoMap.put(EntityType.WIDGET_TYPE, widgetTypeDao);
resourceContainerDaoMap.put(EntityType.DASHBOARD, dashboardInfoDao);
resourceLinkContainerDaoMap.put(EntityType.WIDGET_TYPE, widgetTypeDao);
resourceLinkContainerDaoMap.put(EntityType.DASHBOARD, dashboardInfoDao);
generalResourceContainerDaoMap.put(EntityType.RULE_CHAIN, ruleChainDao);
}
@Autowired @Lazy
@ -206,6 +212,12 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
return resourceDao.getResourceData(tenantId, resourceId);
}
@Override
public TbResourceDataInfo getResourceDataInfo(TenantId tenantId, TbResourceId resourceId) {
log.trace("Executing getResourceDataInfo [{}] [{}]", tenantId, resourceId);
return resourceDao.getResourceDataInfo(tenantId, resourceId);
}
@Override
public ResourceExportData exportResource(TbResourceInfo resourceInfo) {
byte[] data = getResourceData(resourceInfo.getTenantId(), resourceInfo.getId());
@ -344,32 +356,48 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
}
if (!force) {
if (resource.getResourceType() == ResourceType.JS_MODULE) {
var link = resource.getLink();
Map<String, List<? extends HasId<?>>> affectedEntities = new HashMap<>();
resourceContainerDaoMap.forEach((entityType, resourceContainerDao) -> {
var entities = tenantId.isSysTenantId() ? resourceContainerDao.findByResourceLink(link, MAX_ENTITIES_TO_FIND) :
resourceContainerDao.findByTenantIdAndResourceLink(tenantId, link, MAX_ENTITIES_TO_FIND);
if (!entities.isEmpty()) {
affectedEntities.put(entityType.name(), entities);
}
});
if (!affectedEntities.isEmpty()) {
success = false;
result.references(affectedEntities);
}
Map<String, List<EntityInfo>> references = findResourceReferences(tenantId, resource);
if (!references.isEmpty()) {
success = false;
result.references(references);
}
}
if (success) {
resourceDao.removeById(tenantId, resourceId.getId());
publishEvictEvent(new ResourceInfoEvictEvent(tenantId, resourceId));
eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entity(resource).entityId(resourceId).build());
}
return result.success(success).build();
}
private Map<String, List<EntityInfo>> findResourceReferences(TenantId tenantId, TbResourceInfo resource) {
Map<String, List<EntityInfo>> references = new HashMap<>();
if (resource.getResourceType() == ResourceType.JS_MODULE) {
var ref = resource.getLink();
findReferences(tenantId, references, ref, resourceLinkContainerDaoMap);
}
if (resource.getResourceType() == ResourceType.GENERAL) {
var ref = resource.getId().getId().toString();
findReferences(tenantId, references, ref, generalResourceContainerDaoMap);
}
return references;
}
private void findReferences(TenantId tenantId, Map<String, List<EntityInfo>> references, String ref, Map<EntityType, ResourceContainerDao<?>> resourceLinkContainerDaoMap) {
resourceLinkContainerDaoMap.forEach((entityType, dao) -> {
List<EntityInfo> entities = tenantId.isSysTenantId()
? dao.findByResource(ref, MAX_ENTITIES_TO_FIND)
: dao.findByTenantIdAndResource(tenantId, ref, MAX_ENTITIES_TO_FIND);
if (!entities.isEmpty()) {
references.put(entityType.name(), entities);
}
});
}
@Override
public void deleteEntity(TenantId tenantId, EntityId id, boolean force) {
deleteResource(tenantId, (TbResourceId) id, force);
@ -663,6 +691,12 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
return saveResource(resource);
}
@Override
public List<TbResourceInfo> findSystemOrTenantResourcesByIds(TenantId tenantId, List<TbResourceId> resourceIds) {
log.trace("Executing findSystemOrTenantResourcesByIds, tenantId [{}], resourceIds [{}]", tenantId, resourceIds);
return resourceInfoDao.findSystemOrTenantResourcesByIds(tenantId, resourceIds);
}
@Override
public String calculateEtag(byte[] data) {
return Hashing.sha256().hashBytes(data).toString();

72
dao/src/main/java/org/thingsboard/server/dao/resource/DefaultTbResourceDataCache.java

@ -0,0 +1,72 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.resource;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.util.concurrent.FluentFuture;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.sql.JpaExecutorService;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@Slf4j
public class DefaultTbResourceDataCache implements TbResourceDataCache {
private final ResourceService resourceService;
private final JpaExecutorService executorService;
@Value("${cache.tbResourceData.maxSize:100000}")
private int cacheMaxSize;
@Value("${cache.tbResourceData.timeToLiveInMinutes:44640}")
private int cacheValueTtl;
private AsyncLoadingCache<ResourceDataKey, TbResourceDataInfo> cache;
@PostConstruct
private void init() {
cache = Caffeine.newBuilder()
.maximumSize(cacheMaxSize)
.expireAfterAccess(cacheValueTtl, TimeUnit.MINUTES)
.executor(executorService)
.buildAsync((key, executor) -> CompletableFuture.supplyAsync(() -> resourceService.getResourceDataInfo(key.tenantId(), key.resourceId()), executor));
}
@Override
public FluentFuture<TbResourceDataInfo> getResourceDataInfoAsync(TenantId tenantId, TbResourceId resourceId) {
log.trace("Retrieving resource data info by id [{}], tenant id [{}] from cache", resourceId, tenantId);
return DonAsynchron.toFluentFuture(cache.get(new ResourceDataKey(tenantId, resourceId)));
}
@Override
public void evictResourceData(TenantId tenantId, TbResourceId resourceId) {
cache.asMap().remove(new ResourceDataKey(tenantId, resourceId));
log.trace("Evicted resource data info with id [{}], tenant id [{}]", resourceId, tenantId);
}
record ResourceDataKey (TenantId tenantId, TbResourceId resourceId) {}
}

2
dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceDao.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.resource;
import org.thingsboard.server.common.data.ResourceSubType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
@ -51,4 +52,5 @@ public interface TbResourceDao extends Dao<TbResource>, TenantEntityWithDataDao,
long getResourceSize(TenantId tenantId, TbResourceId resourceId);
TbResourceDataInfo getResourceDataInfo(TenantId tenantId, TbResourceId resourceId);
}

2
dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceInfoDao.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.resource;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.TbResourceInfoFilter;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@ -46,4 +47,5 @@ public interface TbResourceInfoDao extends Dao<TbResourceInfo> {
TbResourceInfo findPublicResourceByKey(ResourceType resourceType, String publicResourceKey);
List<TbResourceInfo> findSystemOrTenantResourcesByIds(TenantId tenantId, List<TbResourceId> resourceIds);
}

4
dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java

@ -21,8 +21,10 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.ExportableEntityDao;
import org.thingsboard.server.dao.ResourceContainerDao;
import org.thingsboard.server.dao.TenantEntityDao;
import java.util.Collection;
@ -31,7 +33,7 @@ import java.util.UUID;
/**
* Created by igor on 3/12/18.
*/
public interface RuleChainDao extends Dao<RuleChain>, TenantEntityDao<RuleChain>, ExportableEntityDao<RuleChainId, RuleChain> {
public interface RuleChainDao extends Dao<RuleChain>, TenantEntityDao<RuleChain>, ExportableEntityDao<RuleChainId, RuleChain>, ResourceContainerDao<RuleChain> {
/**
* Find rule chains by tenantId and page link.

6
dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.cf;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity;
@ -36,7 +37,10 @@ public interface CalculatedFieldRepository extends JpaRepository<CalculatedField
Page<CalculatedFieldEntity> findAllByTenantId(UUID tenantId, Pageable pageable);
Page<CalculatedFieldEntity> findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable);
@Query("SELECT cf FROM CalculatedFieldEntity cf WHERE cf.tenantId = :tenantId " +
"AND cf.entityId = :entityId " +
"AND (:textSearch IS NULL OR ilike(cf.name, CONCAT('%', :textSearch, '%')) = true)")
Page<CalculatedFieldEntity> findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, String textSearch, Pageable pageable);
List<CalculatedFieldEntity> findAllByTenantId(UUID tenantId);

2
dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java

@ -85,7 +85,7 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao<CalculatedFieldEntity,
@Override
public PageData<CalculatedField> findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) {
log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink);
return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), DaoUtil.toPageable(pageLink)));
return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink)));
}
@Override

16
dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java

@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.dao.model.sql.DashboardInfoEntity;
import java.util.List;
@ -87,12 +88,15 @@ public interface DashboardInfoRepository extends JpaRepository<DashboardInfoEnti
)
List<DashboardInfoEntity> findByImageLink(@Param("imageLink") String imageLink, @Param("limit") int limit);
@Query(value = "SELECT * FROM dashboard d WHERE d.tenant_id = :tenantId and d.configuration ILIKE CONCAT('%', :link, '%') limit :limit",
nativeQuery = true)
List<DashboardInfoEntity> findDashboardInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, @Param("link") String link, @Param("limit") int limit);
@Query("SELECT new org.thingsboard.server.common.data.EntityInfo(d.id, 'DASHBOARD', d.title) " +
"FROM DashboardEntity d WHERE d.tenantId = :tenantId AND ilike(cast(d.configuration as string), CONCAT('%', :link, '%')) = true")
List<EntityInfo> findDashboardInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId,
@Param("link") String link,
Pageable pageable);
@Query(value = "SELECT * FROM dashboard d WHERE d.configuration ILIKE CONCAT('%', :link, '%') limit :limit",
nativeQuery = true)
List<DashboardInfoEntity> findDashboardInfosByResourceLink(@Param("link") String link, @Param("limit") int limit);
@Query("SELECT new org.thingsboard.server.common.data.EntityInfo(d.id, 'DASHBOARD', d.title) " +
"FROM DashboardEntity d WHERE ilike(cast(d.configuration as string), CONCAT('%', :link, '%')) = true")
List<EntityInfo> findDashboardInfosByResourceLink(@Param("link") String link,
Pageable pageable);
}

10
dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java

@ -17,9 +17,11 @@ package org.thingsboard.server.dao.sql.dashboard;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.DashboardInfo;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@ -135,13 +137,13 @@ public class JpaDashboardInfoDao extends JpaAbstractDao<DashboardInfoEntity, Das
}
@Override
public List<DashboardInfo> findByTenantIdAndResourceLink(TenantId tenantId, String url, int limit) {
return DaoUtil.convertDataList(dashboardInfoRepository.findDashboardInfosByTenantIdAndResourceLink(tenantId.getId(), url, limit));
public List<EntityInfo> findByTenantIdAndResource(TenantId tenantId, String reference, int limit) {
return dashboardInfoRepository.findDashboardInfosByTenantIdAndResourceLink(tenantId.getId(), reference, PageRequest.of(0, limit));
}
@Override
public List<DashboardInfo> findByResourceLink(String link, int limit) {
return DaoUtil.convertDataList(dashboardInfoRepository.findDashboardInfosByResourceLink(link, limit));
public List<EntityInfo> findByResource(String reference, int limit) {
return dashboardInfoRepository.findDashboardInfosByResourceLink(reference, PageRequest.of(0, limit));
}
}

6
dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java

@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceSubType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
@ -115,6 +116,11 @@ public class JpaTbResourceDao extends JpaAbstractDao<TbResourceEntity, TbResourc
return resourceRepository.getDataSizeById(resourceId.getId());
}
@Override
public TbResourceDataInfo getResourceDataInfo(TenantId tenantId, TbResourceId resourceId) {
return resourceRepository.getDataInfoById(resourceId.getId());
}
@Override
public Long sumDataSizeByTenantId(TenantId tenantId) {
return resourceRepository.sumDataSizeByTenantId(tenantId.getId());

8
dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceInfoDao.java

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.ResourceSubType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.TbResourceInfoFilter;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@ -40,6 +41,8 @@ import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.thingsboard.server.dao.DaoUtil.toUUIDs;
@Slf4j
@Component
@SqlDao
@ -126,4 +129,9 @@ public class JpaTbResourceInfoDao extends JpaAbstractDao<TbResourceInfoEntity, T
public TbResourceInfo findPublicResourceByKey(ResourceType resourceType, String publicResourceKey) {
return DaoUtil.getData(resourceInfoRepository.findByResourceTypeAndPublicResourceKeyAndIsPublicTrue(resourceType.name(), publicResourceKey));
}
@Override
public List<TbResourceInfo> findSystemOrTenantResourcesByIds(TenantId tenantId, List<TbResourceId> resourceIds) {
return DaoUtil.convertDataList(resourceInfoRepository.findSystemOrTenantResourcesByIdIn(tenantId.getId(), TenantId.NULL_UUID, toUUIDs(resourceIds)));
}
}

6
dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java

@ -79,4 +79,10 @@ public interface TbResourceInfoRepository extends JpaRepository<TbResourceInfoEn
TbResourceInfoEntity findByResourceTypeAndPublicResourceKeyAndIsPublicTrue(String resourceType, String publicResourceKey);
@Query("SELECT tr FROM TbResourceInfoEntity tr WHERE " +
"tr.id IN (:resourceIds) AND (tr.tenantId = :tenantId OR tr.tenantId = :systemTenantId)")
List<TbResourceInfoEntity> findSystemOrTenantResourcesByIdIn(@Param("tenantId") UUID tenantId,
@Param("systemTenantId") UUID systemTenantId,
@Param("resourceIds") List<UUID> resourceIds);
}

3
dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceRepository.java

@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.thingsboard.server.common.data.TbResourceDataInfo;
import org.thingsboard.server.dao.ExportableEntityRepository;
import org.thingsboard.server.dao.model.sql.TbResourceEntity;
@ -101,4 +102,6 @@ public interface TbResourceRepository extends JpaRepository<TbResourceEntity, UU
@Query("SELECT r.id FROM TbResourceInfoEntity r WHERE r.tenantId = :tenantId")
Page<UUID> findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable);
@Query("SELECT new org.thingsboard.server.common.data.TbResourceDataInfo(r.data, r.descriptor) FROM TbResourceEntity r WHERE r.id = :id")
TbResourceDataInfo getDataInfoById(UUID id);
}

12
dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java

@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.rule;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Limit;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edqs.fields.RuleChainFields;
import org.thingsboard.server.common.data.id.RuleChainId;
@ -141,6 +143,16 @@ public class JpaRuleChainDao extends JpaAbstractDao<RuleChainEntity, RuleChain>
return findRuleChainsByTenantId(tenantId.getId(), pageLink);
}
@Override
public List<EntityInfo> findByTenantIdAndResource(TenantId tenantId, String reference, int limit) {
return ruleChainRepository.findRuleChainsByTenantIdAndResource(tenantId.getId(), reference, PageRequest.of(0, limit));
}
@Override
public List<EntityInfo> findByResource(String reference, int limit) {
return ruleChainRepository.findRuleChainsByResource(reference, PageRequest.of(0, limit));
}
@Override
public List<RuleChainFields> findNextBatch(UUID id, int batchSize) {
return ruleChainRepository.findNextBatch(id, Limit.of(batchSize));

15
dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java

@ -17,10 +17,12 @@ package org.thingsboard.server.dao.sql.rule;
import org.springframework.data.domain.Limit;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.edqs.fields.RuleChainFields;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.dao.ExportableEntityRepository;
@ -72,6 +74,19 @@ public interface RuleChainRepository extends JpaRepository<RuleChainEntity, UUID
@Query("SELECT externalId FROM RuleChainEntity WHERE id = :id")
UUID getExternalIdById(@Param("id") UUID id);
@Query("SELECT new org.thingsboard.server.common.data.EntityInfo(rc.id, 'RULE_CHAIN', rc.name) " +
"FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId AND EXISTS " +
"(SELECT 1 FROM RuleNodeEntity rn WHERE rn.ruleChainId = rc.id AND cast(rn.configuration as string) LIKE CONCAT('%', :resourceId, '%'))")
List<EntityInfo> findRuleChainsByTenantIdAndResource(@Param("tenantId") UUID tenantId,
@Param("resourceId") String resourceId,
PageRequest of);
@Query("SELECT new org.thingsboard.server.common.data.EntityInfo(rc.id, 'RULE_CHAIN', rc.name) " +
"FROM RuleChainEntity rc WHERE EXISTS " +
"(SELECT 1 FROM RuleNodeEntity rn WHERE rn.ruleChainId = rc.id AND cast(rn.configuration as string) LIKE CONCAT('%', :resourceId, '%'))")
List<EntityInfo> findRuleChainsByResource(@Param("resourceId") String resourceId,
Pageable pageable);
@Query("SELECT new org.thingsboard.server.common.data.edqs.fields.RuleChainFields(r.id, r.createdTime, r.tenantId," +
"r.name, r.version, r.additionalInfo) FROM RuleChainEntity r WHERE r.id > :id ORDER BY r.id")
List<RuleChainFields> findNextBatch(@Param("id") UUID id, Limit limit);

10
dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java

@ -17,8 +17,10 @@ package org.thingsboard.server.dao.sql.widget;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Limit;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields;
import org.thingsboard.server.common.data.id.TenantId;
@ -269,13 +271,13 @@ public class JpaWidgetTypeDao extends JpaAbstractDao<WidgetTypeDetailsEntity, Wi
}
@Override
public List<WidgetTypeInfo> findByTenantIdAndResourceLink(TenantId tenantId, String link, int limit) {
return DaoUtil.convertDataList(widgetTypeInfoRepository.findWidgetTypeInfosByTenantIdAndResourceLink(tenantId.getId(), link, limit));
public List<EntityInfo> findByTenantIdAndResource(TenantId tenantId, String reference, int limit) {
return widgetTypeInfoRepository.findWidgetTypeInfosByTenantIdAndResourceLink(tenantId.getId(), reference, PageRequest.of(0, limit));
}
@Override
public List<WidgetTypeInfo> findByResourceLink(String link, int limit) {
return DaoUtil.convertDataList(widgetTypeInfoRepository.findWidgetTypeInfosByResourceLink(link, limit));
public List<EntityInfo> findByResource(String reference, int limit) {
return widgetTypeInfoRepository.findWidgetTypeInfosByResourceLink(reference, PageRequest.of(0, limit));
}
@Override

15
dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java

@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.dao.model.sql.WidgetTypeInfoEntity;
import java.util.List;
@ -214,10 +215,14 @@ public interface WidgetTypeInfoRepository extends JpaRepository<WidgetTypeInfoEn
)
List<WidgetTypeInfoEntity> findByImageUrl(@Param("imageLink") String imageLink, @Param("limit") int limit);
@Query(value = "SELECT * FROM widget_type_info_view w WHERE w.tenant_id = :tenantId AND w.descriptor ILIKE CONCAT('%', :link, '%') LIMIT :limit ", nativeQuery = true)
List<WidgetTypeInfoEntity> findWidgetTypeInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, @Param("link") String link, @Param("limit") int limit);
@Query(value = "SELECT * FROM widget_type_info_view w WHERE w.descriptor ILIKE CONCAT('%', :link, '%') LIMIT :limit ", nativeQuery = true)
List<WidgetTypeInfoEntity> findWidgetTypeInfosByResourceLink(@Param("link") String link, @Param("limit") int limit);
@Query("SELECT new org.thingsboard.server.common.data.EntityInfo(w.id, 'WIDGET_TYPE', w.name) " +
"FROM WidgetTypeEntity w WHERE w.tenantId = :tenantId AND ilike(cast(w.descriptor as string), CONCAT('%', :link, '%')) = true")
List<EntityInfo> findWidgetTypeInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId,
@Param("link") String link,
Pageable pageable);
@Query("SELECT new org.thingsboard.server.common.data.EntityInfo(w.id, 'WIDGET_TYPE', w.name) " +
"FROM WidgetTypeEntity w WHERE ilike(cast(w.descriptor as string), CONCAT('%', :link, '%')) = true")
List<EntityInfo> findWidgetTypeInfosByResourceLink(@Param("link") String link,
Pageable pageable);
}

29
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@ -113,6 +113,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
@Value("${cassandra.query.use_ts_key_value_partitioning_on_read:true}")
private boolean useTsKeyValuePartitioningOnRead;
@Getter
@Value("${cassandra.query.use_ts_key_value_partitioning_on_read_max_estimated_partition_count:40}") // 3+ years for MONTHS
private int useTsKeyValuePartitioningOnReadMaxEstimatedPartitionCount;
@Value("${cassandra.query.ts_key_value_partitions_max_cache_size:100000}")
private long partitionsCacheSize;
@ -415,22 +419,41 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
readResultsProcessingExecutor);
}
private ListenableFuture<List<Long>> getPartitionsFuture(TenantId tenantId, TsKvQuery query, EntityId entityId, long minPartition, long maxPartition) {
ListenableFuture<List<Long>> getPartitionsFuture(TenantId tenantId, TsKvQuery query, EntityId entityId, long minPartition, long maxPartition) {
if (isFixedPartitioning()) { //no need to fetch partitions from DB
return Futures.immediateFuture(FIXED_PARTITION);
}
if (!isUseTsKeyValuePartitioningOnRead()) {
return Futures.immediateFuture(calculatePartitions(minPartition, maxPartition));
final long estimatedPartitionCount = estimatePartitionCount(minPartition, maxPartition);
if (estimatedPartitionCount <= useTsKeyValuePartitioningOnReadMaxEstimatedPartitionCount) {
return Futures.immediateFuture(calculatePartitions(minPartition, maxPartition, (int) estimatedPartitionCount));
}
}
return getPartitionsFromDB(tenantId, query, entityId, minPartition, maxPartition);
}
ListenableFuture<List<Long>> getPartitionsFromDB(TenantId tenantId, TsKvQuery query, EntityId entityId, long minPartition, long maxPartition) {
TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition);
return Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor);
}
// Optimistic estimation of partition count, expected to be never called for infinite partitioning
long estimatePartitionCount(long minPartition, long maxPartition) {
if (maxPartition > minPartition) {
return (maxPartition - minPartition) / tsFormat.getDurationMs() + 2; //at least 2 partitions, at max 2 partitions overestimated
}
return 1; // 1 or 0, but 1 is more optimistic
}
List<Long> calculatePartitions(long minPartition, long maxPartition) {
return calculatePartitions(minPartition, maxPartition, 0);
}
List<Long> calculatePartitions(long minPartition, long maxPartition, int estimatedPartitionCount) {
if (minPartition == maxPartition) {
return Collections.singletonList(minPartition);
}
List<Long> partitions = new ArrayList<>();
List<Long> partitions = estimatedPartitionCount > 0 ? new ArrayList<>(estimatedPartitionCount) : new ArrayList<>();
long currentPartition = minPartition;
LocalDateTime currentPartitionTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(currentPartition), ZoneOffset.UTC);

15
dao/src/main/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDate.java

@ -15,32 +15,29 @@
*/
package org.thingsboard.server.dao.timeseries;
import lombok.Getter;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalUnit;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@Getter
public enum NoSqlTsPartitionDate {
MINUTES("yyyy-MM-dd-HH-mm", ChronoUnit.MINUTES), HOURS("yyyy-MM-dd-HH", ChronoUnit.HOURS), DAYS("yyyy-MM-dd", ChronoUnit.DAYS), MONTHS("yyyy-MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS),INDEFINITE("",ChronoUnit.FOREVER);
private final String pattern;
private final transient TemporalUnit truncateUnit;
private final transient long durationMs;
public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0,0, ZoneOffset.UTC);
NoSqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) {
this.pattern = pattern;
this.truncateUnit = truncateUnit;
}
public String getPattern() {
return pattern;
}
public TemporalUnit getTruncateUnit() {
return truncateUnit;
this.durationMs = TimeUnit.SECONDS.toMillis(this.truncateUnit.getDuration().getSeconds());
}
public LocalDateTime truncatedTo(LocalDateTime time) {

1
dao/src/test/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDaoTest.java

@ -24,6 +24,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;

12
dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMinutesAlwaysExistsTest.java

@ -117,4 +117,16 @@ public class CassandraBaseTimeseriesDaoPartitioningMinutesAlwaysExistsTest {
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:10:00Z").getTime()));
}
@Test
public void testEstimatePartitionCount() throws ParseException {
assertThat(tsDao.estimatePartitionCount(0, Long.MAX_VALUE)).as("centuries").isEqualTo(153_722_867_280_914L);
assertThat(tsDao.estimatePartitionCount(0, 0)).as("single").isEqualTo(1L);
long startTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-12T00:00:00Z").getTime());
long endTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-31T23:59:59Z").getTime());
assertThat(tsDao.estimatePartitionCount(startTs, endTs)).as("600,479 minutes + 2 spare periods").isEqualTo(600479 + 2);
assertThat(tsDao.estimatePartitionCount(endTs, startTs)).as("wrong period estimated as 1").isEqualTo(1L);
}
}

63
dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest.java

@ -15,25 +15,35 @@
*/
package org.thingsboard.server.dao.timeseries;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.bean.override.mockito.MockitoSpyBean;
import org.springframework.test.context.junit4.SpringRunner;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.TsKvQuery;
import org.thingsboard.server.dao.cassandra.CassandraCluster;
import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor;
import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor;
import java.text.ParseException;
import java.util.List;
import java.util.UUID;
import static org.apache.commons.lang3.time.DateFormatUtils.ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = CassandraBaseTimeseriesDao.class)
@ -50,7 +60,7 @@ import static org.assertj.core.api.Assertions.assertThat;
@Slf4j
public class CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest {
@Autowired
@MockitoSpyBean
CassandraBaseTimeseriesDao tsDao;
@MockBean(answer = Answers.RETURNS_MOCKS)
@ -131,4 +141,53 @@ public class CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest {
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-01T00:00:00Z").getTime()));
}
@Test
public void testEstimatePartitionCount() throws ParseException {
assertThat(tsDao.estimatePartitionCount(0, Long.MAX_VALUE)).as("centuries").isEqualTo(3_507_324_297L);
assertThat(tsDao.estimatePartitionCount(0, 0)).as("single").isEqualTo(1L);
long startTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-12T00:00:00Z").getTime());
long endTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-31T23:59:59Z").getTime());
assertThat(tsDao.estimatePartitionCount(startTs, endTs)).as("13 month + 2 spare periods").isEqualTo(13 + 2);
assertThat(tsDao.estimatePartitionCount(endTs, startTs)).as("wrong period estimated as 1").isEqualTo(1L);
}
@Test
public void testGetPartitionsFutureModeratePartitionsCount() throws ParseException {
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
TsKvQuery query = mock(TsKvQuery.class);
long startTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-12T00:00:00Z").getTime());
long endTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-31T23:59:59Z").getTime());
willReturn(mock(ListenableFuture.class)).given(tsDao).getPartitionsFromDB(tenantId, query, tenantId, startTs, endTs);
tsDao.getPartitionsFuture(tenantId, query, tenantId, startTs, endTs);
verify(tsDao).estimatePartitionCount(startTs, endTs);
verify(tsDao).calculatePartitions(eq(startTs), eq(endTs), anyInt());
verify(tsDao, never()).getPartitionsFromDB(tenantId, query, tenantId, startTs, endTs);
}
@Test
public void testGetPartitionsFutureHugePartitionsCountPreventOOMFallbackToDB() throws ParseException {
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
TsKvQuery query = mock(TsKvQuery.class);
long startTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2000-12-12T00:00:00Z").getTime());
long endTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("3000-01-31T23:59:59Z").getTime());
willReturn(mock(ListenableFuture.class)).given(tsDao).getPartitionsFromDB(tenantId, query, tenantId, startTs, endTs);
tsDao.getPartitionsFuture(tenantId, query, tenantId, startTs, endTs);
verify(tsDao).estimatePartitionCount(startTs, endTs);
verify(tsDao, never()).calculatePartitions(eq(startTs), eq(endTs), anyInt());
verify(tsDao).getPartitionsFromDB(tenantId, query, tenantId, startTs, endTs);
}
}

12
dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningYearsAlwaysExistsTest.java

@ -112,4 +112,16 @@ public class CassandraBaseTimeseriesDaoPartitioningYearsAlwaysExistsTest {
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2025-01-01T00:00:00Z").getTime()));
}
@Test
public void testEstimatePartitionCount() throws ParseException {
assertThat(tsDao.estimatePartitionCount(0, Long.MAX_VALUE)).as("centuries").isEqualTo(292_277_026L);
assertThat(tsDao.estimatePartitionCount(0, 0)).as("single").isEqualTo(1L);
long startTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-12T00:00:00Z").getTime());
long endTs = tsDao.toPartitionTs(
ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-31T23:59:59Z").getTime());
assertThat(tsDao.estimatePartitionCount(startTs, endTs)).as("2 years + 2 spare periods").isEqualTo(2 + 2);
assertThat(tsDao.estimatePartitionCount(endTs, startTs)).as("wrong period estimated as 1").isEqualTo(1L);
}
}

41
dao/src/test/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDateTest.java

@ -0,0 +1,41 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.timeseries;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import static org.assertj.core.api.Assertions.assertThat;
class NoSqlTsPartitionDateTest {
@ParameterizedTest
@EnumSource(NoSqlTsPartitionDate.class)
void getDurationMsTest(NoSqlTsPartitionDate tsPartitionDate) throws Exception {
final Long durationMs = switch (tsPartitionDate) {
case MINUTES -> 60000L;
case HOURS -> 3600000L;
case DAYS -> 86400000L;
case MONTHS -> 2629746000L;
case YEARS -> 31556952000L;
case INDEFINITE -> Long.MAX_VALUE;
default -> null; //should be here in case a new enum value will be added in future
};
assertThat(durationMs).isNotNull();
assertThat(tsPartitionDate.getDurationMs()).isEqualTo(durationMs);
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save