From f81413112c2858e44dc308c056c38f8fbc5ed94f Mon Sep 17 00:00:00 2001 From: Artem Babak Date: Wed, 26 May 2021 21:02:35 +0300 Subject: [PATCH 001/101] Edge: menu order changes --- ui-ngx/src/app/core/services/menu.service.ts | 32 +++++++++---------- .../pages/customer/customer-routing.module.ts | 2 +- .../customers-table-config.resolver.ts | 2 +- .../dashboards-table-config.resolver.ts | 2 +- .../home/pages/edge/edge-routing.module.ts | 26 +++++++++++++-- .../pages/edge/edges-table-config.resolver.ts | 2 +- .../rulechain/rule-node-details.component.ts | 2 +- .../rulechain/rulechain-page.component.ts | 2 +- .../rulechains-table-config.resolver.ts | 6 ++-- .../pages/rulechain/rulenode.component.ts | 2 +- 10 files changed, 49 insertions(+), 29 deletions(-) diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 07a6d8f237..10dedd4fc7 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -291,26 +291,26 @@ export class MenuService { ); if (authState.edgesSupportEnabled) { sections.push( + { + id: guid(), + name: 'edge.edge-instances', + type: 'link', + path: '/edgeInstances', + icon: 'router' + }, { id: guid(), name: 'edge.management', type: 'toggle', - path: '/edges', - height: '80px', - icon: 'router', + path: '/edgeManagement', + height: '40px', + icon: 'settings_input_antenna', pages: [ - { - id: guid(), - name: 'edge.edge-instances', - type: 'link', - path: '/edges', - icon: 'router' - }, { id: guid(), name: 'edge.rulechain-templates', type: 'link', - path: '/edges/ruleChains', + path: '/edgeManagement/ruleChains', icon: 'settings_ethernet' } ] @@ -448,12 +448,12 @@ export class MenuService { { name: 'edge.edge-instances', icon: 'router', - path: '/edges' + path: '/edgeInstances' }, { name: 'edge.rulechain-templates', icon: 'settings_ethernet', - path: '/edges/ruleChains' + path: '/edgeManagement/ruleChains' } ] } @@ -548,7 +548,7 @@ export class MenuService { id: guid(), name: 'edge.edge-instances', type: 'link', - path: '/edges', + path: '/edgeInstances', icon: 'router' } ); @@ -606,8 +606,8 @@ export class MenuService { places: [ { name: 'edge.edge-instances', - icon: 'router', - path: '/edges' + icon: 'settings_input_antenna', + path: '/edgeInstances' } ] } diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts b/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts index f5f3ee05fa..2ed306c2d8 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts @@ -98,7 +98,7 @@ const routes: Routes = [ } }, { - path: ':customerId/edges', + path: ':customerId/edgeInstances', component: EntitiesTableComponent, data: { auth: [Authority.TENANT_ADMIN], diff --git a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts index 8ac9e64390..5a6e03b3d7 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts @@ -169,7 +169,7 @@ export class CustomersTableConfigResolver implements Resolve): boolean { diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts index c572fa958f..a4c17e75ac 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts @@ -362,7 +362,7 @@ export class DashboardsTableConfigResolver implements Resolve) { diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts index 00cd38b9c4..d5a3fcc8a0 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts @@ -152,7 +152,7 @@ export class RuleNodeDetailsComponent extends PageComponent implements OnInit, O } if (this.ruleNode.targetRuleChainId) { if (this.ruleChainType === RuleChainType.EDGE) { - this.router.navigateByUrl(`/edges/ruleChains/${this.ruleNode.targetRuleChainId}`); + this.router.navigateByUrl(`/edgeManagement/ruleChains/${this.ruleNode.targetRuleChainId}`); } else { this.router.navigateByUrl(`/ruleChains/${this.ruleNode.targetRuleChainId}`); } diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts index 54102ffbcd..ea9cc54747 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts @@ -1289,7 +1289,7 @@ export class RuleChainPageComponent extends PageComponent if (this.ruleChainType !== RuleChainType.EDGE) { this.router.navigateByUrl(`ruleChains/${this.ruleChain.id.id}`); } else { - this.router.navigateByUrl(`edges/ruleChains/${this.ruleChain.id.id}`); + this.router.navigateByUrl(`edgeManagement/ruleChains/${this.ruleChain.id.id}`); } } else { this.createRuleChainModel(); diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts index 37a21cab2e..ffc05052fe 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts @@ -285,7 +285,7 @@ export class RuleChainsTableConfigResolver implements Resolve Date: Thu, 27 May 2021 09:39:53 +0300 Subject: [PATCH 002/101] Minor code cleanup --- .../src/app/modules/home/pages/edge/edge-routing.module.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts b/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts index bd96b953a8..4ffa51a6bd 100644 --- a/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts @@ -39,7 +39,6 @@ import { RuleNodeComponentsResolver, TooltipsterResolver } from '@home/pages/rulechain/rulechain-routing.module'; -import { CustomersTableConfigResolver } from '@home/pages/customer/customers-table-config.resolver'; const routes: Routes = [ { @@ -56,7 +55,7 @@ const routes: Routes = [ component: EntitiesTableComponent, data: { auth: [Authority.TENANT_ADMIN, Authority.CUSTOMER_USER], - edgesType: 'tenant', + edgesType: 'tenant' }, resolve: { entitiesTableConfig: EdgesTableConfigResolver @@ -272,8 +271,7 @@ const routes: Routes = [ ] } ] - } - ]; + }]; @NgModule({ imports: [RouterModule.forChild(routes)], From 0c886656543987f27d59c2c053c11677de147a8b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 25 May 2021 19:44:34 +0300 Subject: [PATCH 003/101] js-executor: scriptMap refactored from Object to the Map() --- .../api/jsInvokeMessageProcessor.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 2964291a30..a760e21e47 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -34,10 +34,9 @@ const slowQueryLogBody = config.get('script.slow_query_log_body') === 'true'; const {performance} = require('perf_hooks'); function JsInvokeMessageProcessor(producer) { - console.log("Producer:", producer); this.producer = producer; this.executor = new JsExecutor(useSandbox); - this.scriptMap = {}; + this.scriptMap = new Map(); this.scriptIds = []; this.executedScriptsCounter = 0; } @@ -157,12 +156,12 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) { var scriptId = getScriptId(releaseRequest); logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); - if (this.scriptMap[scriptId]) { + if (this.scriptMap.has(scriptId)) { var index = this.scriptIds.indexOf(scriptId); if (index > -1) { this.scriptIds.splice(index, 1); } - delete this.scriptMap[scriptId]; + this.scriptMap.delete(scriptId); } var releaseResponse = createReleaseResponse(scriptId, true); logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId); @@ -189,8 +188,8 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { var self = this; return new Promise(function(resolve, reject) { - if (self.scriptMap[scriptId]) { - resolve(self.scriptMap[scriptId]); + if (self.scriptMap.has(scriptId)) { + resolve(self.scriptMap.get(scriptId)); } else { self.executor.compileScript(scriptBody).then( (script) => { @@ -206,16 +205,17 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scrip } JsInvokeMessageProcessor.prototype.cacheScript = function(scriptId, script) { - if (!this.scriptMap[scriptId]) { + if (!this.scriptMap.has(scriptId)) { this.scriptIds.push(scriptId); while (this.scriptIds.length > maxActiveScripts) { logger.info('Active scripts count [%s] exceeds maximum limit [%s]', this.scriptIds.length, maxActiveScripts); const prevScriptId = this.scriptIds.shift(); logger.info('Removing active script with id [%s]', prevScriptId); - delete this.scriptMap[prevScriptId]; + this.scriptMap.delete(prevScriptId); } } - this.scriptMap[scriptId] = script; + this.scriptMap.set(scriptId, script); + logger.info("scriptMap size is [%s]", this.scriptMap.size); } function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) { From d729d9ee959963b570aeeaf27787cdd6b6f44bdf Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 27 May 2021 14:50:42 +0300 Subject: [PATCH 004/101] js-executor: instrumentation event for producer and consumer to define the exact flow how to Kafka works without batches (for debug only) --- msa/js-executor/queue/kafkaTemplate.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 2f4fd8751d..1fd21185d0 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -114,6 +114,30 @@ function KafkaProducer() { consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); producer = kafkaClient.producer(); + + const { CONNECT } = producer.events; + const removeListenerC = producer.on(CONNECT, e => logger.info(`producer CONNECT`)); + const { DISCONNECT } = producer.events; + const removeListenerD = producer.on(DISCONNECT, e => logger.info(`producer DISCONNECT`)); + const { REQUEST } = producer.events; + const removeListenerR = producer.on(REQUEST, e => logger.info(`producer REQUEST ${e.payload.broker}`)); + const { REQUEST_TIMEOUT } = producer.events; + const removeListenerRT = producer.on(REQUEST_TIMEOUT, e => logger.info(`producer REQUEST_TIMEOUT ${e.payload.broker}`)); + const { REQUEST_QUEUE_SIZE } = producer.events; + const removeListenerRQS = producer.on(REQUEST_QUEUE_SIZE, e => logger.info(`producer REQUEST_QUEUE_SIZE ${e.payload.broker} size ${e.queueSize}`)); + + const removeListeners = {} + const { FETCH_START } = consumer.events; + removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); + const { FETCH } = consumer.events; + removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); + const { START_BATCH_PROCESS } = consumer.events; + removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { END_BATCH_PROCESS } = consumer.events; + removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { COMMIT_OFFSETS } = consumer.events; + removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); + const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); From 35e2ff99c3d3f7a488fdc4d3fb3375daa06d41f9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 27 May 2021 14:55:10 +0300 Subject: [PATCH 005/101] js-executor: send messages as batch --- .../api/jsInvokeMessageProcessor.js | 25 ++++---- msa/js-executor/queue/kafkaTemplate.js | 63 ++++++++++++++----- 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index a760e21e47..867dd82b39 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -172,17 +172,20 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT var tStartSending = performance.now(); var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); - this.producer.send(responseTopic, scriptId, rawResponse, headers).then( - () => { - logger.debug('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); - }, - (err) => { - if (err) { - logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); - logger.error(err.stack); - } - } - ); + logger.debug('[%s] Sending response to queue, scriptId: [%s]', requestId, scriptId); + this.producer.send(responseTopic, scriptId, rawResponse, headers); +//TODO put error msg for other queues implementation except Kafka +// .then( +// () => { +// logger.info('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); +// }, +// (err) => { +// if (err) { +// logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); +// logger.error(err.stack); +// } +// } +// ); } JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 1fd21185d0..da46a45301 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -33,24 +33,54 @@ let producer; const configEntries = []; +let topicMessages = []; +let loopSend; + function KafkaProducer() { - this.send = async (responseTopic, scriptId, rawResponse, headers) => { - return producer.send( - { - topic: responseTopic, - acks: acks, - compression: compressionType, - messages: [ - { - key: scriptId, - value: rawResponse, - headers: headers.data - } - ] - }); + this.send = (responseTopic, scriptId, rawResponse, headers) => { + logger.debug('Pending queue response, scriptId: [%s]', scriptId); + const message = { + topic: responseTopic, + messages: [{ + key: scriptId, + value: rawResponse, + headers: headers.data + }] + }; + + topicMessages.push(message); + return {}; } } +function sendLoopFunction() { + loopSend = setInterval(sendProducerMsg, 200); +} + +function sendProducerMsg() { + if (topicMessages.length > 0) { + logger.info('sendProducerMsg from queue response, lenght: [%s]', topicMessages.length ); + const messagesToSend = topicMessages; + topicMessages = []; + producer.sendBatch({ + topicMessages: messagesToSend, + acks: acks, + compression: compressionType + }).then( + () => { + logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); + }, + (err) => { + if (err) { + logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); + topicMessages = messagesToSend.concat(topicMessages); + logger.error(err.stack); + } + } + ); + } +} + (async () => { try { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); @@ -141,6 +171,7 @@ function KafkaProducer() { const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); + sendLoopFunction(); await consumer.subscribe({topic: requestTopic}); logger.info('Started ThingsBoard JavaScript Executor Microservice.'); @@ -221,6 +252,10 @@ async function disconnectProducer() { var _producer = producer; producer = null; try { + logger.info('Stopping loop...'); + //TODO: send handle msg + clearInterval(loopSend); + sendProducerMsg(); await _producer.disconnect(); logger.info('Kafka Producer stopped.'); } catch (e) { From 0970ce65b448f622056c070831aa1625f1e81f36 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 27 May 2021 18:45:29 +0300 Subject: [PATCH 006/101] js-executor: maxBatchSize --- msa/js-executor/queue/kafkaTemplate.js | 88 ++++++++++++++++---------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index da46a45301..a061215b28 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -26,6 +26,9 @@ const acks = Number(config.get('kafka.acks')); const requestTimeout = Number(config.get('kafka.requestTimeout')); const compressionType = (config.get('kafka.requestTimeout') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; +const linger = 5; //milliseconds //TODO move to the config +const maxBatchSize = 10; //max messages in batch //TODO move to the config + let kafkaClient; let kafkaAdmin; let consumer; @@ -33,8 +36,8 @@ let producer; const configEntries = []; -let topicMessages = []; -let loopSend; +let batchMessages = []; +let sendLoopInstance; function KafkaProducer() { this.send = (responseTopic, scriptId, rawResponse, headers) => { @@ -48,36 +51,51 @@ function KafkaProducer() { }] }; - topicMessages.push(message); + pushMessageToSendLater(message); return {}; } } -function sendLoopFunction() { - loopSend = setInterval(sendProducerMsg, 200); +function pushMessageToSendLater(message) { + batchMessages.push(message); + if (batchMessages.length >= maxBatchSize) { + sendMessagesAsBatch(); + sendLoopWithLinger(); //reset loop function and reschedule new linger + } +} + +function sendLoopWithLinger() { + if (sendLoopInstance) { + logger.debug("Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); + clearInterval(sendLoopInstance); + } else { + logger.debug("Starting new send loop with linger [%s]", linger) + } + sendLoopInstance = setInterval(sendMessagesAsBatch, linger); } -function sendProducerMsg() { - if (topicMessages.length > 0) { - logger.info('sendProducerMsg from queue response, lenght: [%s]', topicMessages.length ); - const messagesToSend = topicMessages; - topicMessages = []; +function sendMessagesAsBatch() { + if (batchMessages.length > 0) { + logger.info('sendMessagesAsBatch, lenght: [%s]', batchMessages.length ); + const messagesToSend = batchMessages; + batchMessages = []; producer.sendBatch({ topicMessages: messagesToSend, acks: acks, compression: compressionType }).then( - () => { - logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); - }, - (err) => { - if (err) { - logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); - topicMessages = messagesToSend.concat(topicMessages); - logger.error(err.stack); - } - } - ); + () => { + logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); + }, + (err) => { + logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); + batchMessages = messagesToSend.concat(batchMessages); + logger.error(err.stack); + } + ); + + } else { + //logger.debug("nothing to send"); } } @@ -156,22 +174,22 @@ function sendProducerMsg() { const { REQUEST_QUEUE_SIZE } = producer.events; const removeListenerRQS = producer.on(REQUEST_QUEUE_SIZE, e => logger.info(`producer REQUEST_QUEUE_SIZE ${e.payload.broker} size ${e.queueSize}`)); - const removeListeners = {} - const { FETCH_START } = consumer.events; - removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); - const { FETCH } = consumer.events; - removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); - const { START_BATCH_PROCESS } = consumer.events; - removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); - const { END_BATCH_PROCESS } = consumer.events; - removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); - const { COMMIT_OFFSETS } = consumer.events; - removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); +// const removeListeners = {} +// const { FETCH_START } = consumer.events; +// removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); +// const { FETCH } = consumer.events; +// removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); +// const { START_BATCH_PROCESS } = consumer.events; +// removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); +// const { END_BATCH_PROCESS } = consumer.events; +// removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); +// const { COMMIT_OFFSETS } = consumer.events; +// removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); - sendLoopFunction(); + sendLoopWithLinger(); await consumer.subscribe({topic: requestTopic}); logger.info('Started ThingsBoard JavaScript Executor Microservice.'); @@ -254,8 +272,8 @@ async function disconnectProducer() { try { logger.info('Stopping loop...'); //TODO: send handle msg - clearInterval(loopSend); - sendProducerMsg(); + clearInterval(sendLoopInstance); + sendMessagesAsBatch(); await _producer.disconnect(); logger.info('Kafka Producer stopped.'); } catch (e) { From 93bea7020584343511b1a561bb861fd010df037e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 28 May 2021 09:41:35 +0300 Subject: [PATCH 007/101] js-executor: added parameters for producer TB_KAFKA_BATCH_SIZE and TB_KAFKA_LINGER_MS; added print stats frequency SCRIPT_STAT_PRINT_FREQUENCY --- .../api/jsInvokeMessageProcessor.js | 17 +++++--- .../config/custom-environment-variables.yml | 3 ++ msa/js-executor/config/default.yml | 7 ++- msa/js-executor/queue/kafkaTemplate.js | 43 +++++++++++-------- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 867dd82b39..dec935c8ad 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -25,6 +25,7 @@ const config = require('config'), Utils = require('./utils'), JsExecutor = require('./jsExecutor'); +const statFrequency = Number(config.get('script.stat_print_frequency')); const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency')); const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); @@ -39,6 +40,7 @@ function JsInvokeMessageProcessor(producer) { this.scriptMap = new Map(); this.scriptIds = []; this.executedScriptsCounter = 0; + this.lastStatTime = performance.now(); } JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { @@ -118,11 +120,16 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re var scriptId = getScriptId(invokeRequest); logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; - if ( this.executedScriptsCounter >= scriptBodyTraceFrequency ) { - this.executedScriptsCounter = 0; - if (logger.levels[logger.level] >= logger.levels['debug']) { - logger.debug('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody); - } + if (this.executedScriptsCounter % statFrequency == 0) { + var nowMs = performance.now(); + var msSinceLastStat = nowMs - this.lastStatTime; + var requestsPerSec = statFrequency / msSinceLastStat * 1000; //msSinceLastStat can't be zero in the real world + this.lastStatTime = nowMs; + logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s]', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec); + } + + if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) { + logger.info('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody); } this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then( (script) => { diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 5e2bfe8342..c5eb15e8b9 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -26,6 +26,8 @@ kafka: servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge + batch_size: "${TB_KAFKA_BATCH_SIZE:128}" # for producer + linger_ms: "${TB_KAFKA_LINGER_MS:1}" # for producer requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" @@ -70,6 +72,7 @@ logger: script: use_sandbox: "SCRIPT_USE_SANDBOX" + stat_print_frequency: "SCRIPT_STAT_PRINT_FREQUENCY" script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY" max_active_scripts: "MAX_ACTIVE_SCRIPTS" slow_query_log_ms: "SLOW_QUERY_LOG_MS" #1.123456 diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index cdf5b35421..5a2c47a675 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -26,6 +26,8 @@ kafka: servers: "localhost:9092" replication_factor: "1" acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge + batch_size: "128" # for producer + linger_ms: "1" # for producer requestTimeout: "30000" # The default value in kafkajs is: 30000 compression: "gzip" # gzip or uncompressed topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1" @@ -59,7 +61,8 @@ logger: script: use_sandbox: "true" - script_body_trace_frequency: "1000" + script_body_trace_frequency: "10000" + stat_print_frequency: "10000" max_active_scripts: "1000" - slow_query_log_ms: "1.000000" #millis + slow_query_log_ms: "5.000000" #millis slow_query_log_body: "false" diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index a061215b28..79d87c17b4 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -23,12 +23,11 @@ const replicationFactor = Number(config.get('kafka.replication_factor')); const topicProperties = config.get('kafka.topic_properties'); const kafkaClientId = config.get('kafka.client_id'); const acks = Number(config.get('kafka.acks')); +const maxBatchSize = Number(config.get('kafka.batch_size')); +const linger = Number(config.get('kafka.linger_ms')); const requestTimeout = Number(config.get('kafka.requestTimeout')); const compressionType = (config.get('kafka.requestTimeout') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; -const linger = 5; //milliseconds //TODO move to the config -const maxBatchSize = 10; //max messages in batch //TODO move to the config - let kafkaClient; let kafkaAdmin; let consumer; @@ -76,7 +75,9 @@ function sendLoopWithLinger() { function sendMessagesAsBatch() { if (batchMessages.length > 0) { - logger.info('sendMessagesAsBatch, lenght: [%s]', batchMessages.length ); + if (batchMessages.length > 1) { + logger.info('sendMessagesAsBatch, length: [%s]', batchMessages.length); + } const messagesToSend = batchMessages; batchMessages = []; producer.sendBatch({ @@ -85,17 +86,15 @@ function sendMessagesAsBatch() { compression: compressionType }).then( () => { - logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); + logger.debug('Response sent to kafka, length: [%s]', messagesToSend.length); }, (err) => { - logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); + logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); batchMessages = messagesToSend.concat(batchMessages); logger.error(err.stack); } ); - } else { - //logger.debug("nothing to send"); } } @@ -163,6 +162,8 @@ function sendMessagesAsBatch() { consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); producer = kafkaClient.producer(); +/* + //producer event instrumentation to debug const { CONNECT } = producer.events; const removeListenerC = producer.on(CONNECT, e => logger.info(`producer CONNECT`)); const { DISCONNECT } = producer.events; @@ -173,18 +174,22 @@ function sendMessagesAsBatch() { const removeListenerRT = producer.on(REQUEST_TIMEOUT, e => logger.info(`producer REQUEST_TIMEOUT ${e.payload.broker}`)); const { REQUEST_QUEUE_SIZE } = producer.events; const removeListenerRQS = producer.on(REQUEST_QUEUE_SIZE, e => logger.info(`producer REQUEST_QUEUE_SIZE ${e.payload.broker} size ${e.queueSize}`)); +*/ -// const removeListeners = {} -// const { FETCH_START } = consumer.events; -// removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); -// const { FETCH } = consumer.events; -// removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); -// const { START_BATCH_PROCESS } = consumer.events; -// removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); -// const { END_BATCH_PROCESS } = consumer.events; -// removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); -// const { COMMIT_OFFSETS } = consumer.events; -// removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); +/* + //consumer event instrumentation to debug + const removeListeners = {} + const { FETCH_START } = consumer.events; + removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); + const { FETCH } = consumer.events; + removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); + const { START_BATCH_PROCESS } = consumer.events; + removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { END_BATCH_PROCESS } = consumer.events; + removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { COMMIT_OFFSETS } = consumer.events; + removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); +*/ const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); From 2cba1e2f16e9a1a438d2e4bf7af0afa8968fbae5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 28 May 2021 18:07:31 +0300 Subject: [PATCH 008/101] js-executor reduced log severity to debug --- msa/js-executor/queue/kafkaTemplate.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 79d87c17b4..f467d0813b 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -75,9 +75,7 @@ function sendLoopWithLinger() { function sendMessagesAsBatch() { if (batchMessages.length > 0) { - if (batchMessages.length > 1) { - logger.info('sendMessagesAsBatch, length: [%s]', batchMessages.length); - } + logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); const messagesToSend = batchMessages; batchMessages = []; producer.sendBatch({ From a4e28ad94533610c4de0f855bad75dcd4460ac74 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 1 Jun 2021 16:00:32 +0300 Subject: [PATCH 009/101] js-executor fixed promises for each message for Kafka batches --- .../api/jsInvokeMessageProcessor.js | 24 +++++++------- msa/js-executor/queue/kafkaTemplate.js | 33 ++++++++++++------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index dec935c8ad..1aabd216d2 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -180,19 +180,17 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); logger.debug('[%s] Sending response to queue, scriptId: [%s]', requestId, scriptId); - this.producer.send(responseTopic, scriptId, rawResponse, headers); -//TODO put error msg for other queues implementation except Kafka -// .then( -// () => { -// logger.info('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); -// }, -// (err) => { -// if (err) { -// logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); -// logger.error(err.stack); -// } -// } -// ); + this.producer.send(responseTopic, scriptId, rawResponse, headers).then( + () => { + logger.debug('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); + }, + (err) => { + if (err) { + logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); + logger.error(err.stack); + } + } + ); } JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index f467d0813b..1fac350976 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -36,11 +36,12 @@ let producer; const configEntries = []; let batchMessages = []; +let batchResolvers = []; let sendLoopInstance; function KafkaProducer() { this.send = (responseTopic, scriptId, rawResponse, headers) => { - logger.debug('Pending queue response, scriptId: [%s]', scriptId); + logger.debug('Pending queue response, scriptId: [%s]', scriptId); const message = { topic: responseTopic, messages: [{ @@ -50,17 +51,22 @@ function KafkaProducer() { }] }; - pushMessageToSendLater(message); - return {}; + return pushMessageToSendLater(message); } } function pushMessageToSendLater(message) { + let resolver; + const promise = new Promise((resolve, reject) => { + resolver = resolve; + }); batchMessages.push(message); + batchResolvers.push(resolver); if (batchMessages.length >= maxBatchSize) { sendMessagesAsBatch(); sendLoopWithLinger(); //reset loop function and reschedule new linger } + return promise; } function sendLoopWithLinger() { @@ -77,22 +83,27 @@ function sendMessagesAsBatch() { if (batchMessages.length > 0) { logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); const messagesToSend = batchMessages; + const resolvers = batchResolvers; batchMessages = []; + batchResolvers = []; producer.sendBatch({ topicMessages: messagesToSend, acks: acks, compression: compressionType }).then( - () => { - logger.debug('Response sent to kafka, length: [%s]', messagesToSend.length); - }, - (err) => { - logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); - batchMessages = messagesToSend.concat(batchMessages); - logger.error(err.stack); + () => { + logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); + for (let i = 0; i < promisesToSend.length; i++) { + resolvers[i](); } + }, + (err) => { + logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); + logger.error(err.stack); + batchMessages = messagesToSend.concat(batchMessages); + batchResolvers = resolvers.concat(batchResolvers); //promises will never be rejected. Will retry forever + } ); - } } From 1d9fc4a322f8067b6e35bc96e8ff28eb13f2ca8e Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 1 Jun 2021 18:01:21 +0300 Subject: [PATCH 010/101] js-executor: typo fix --- msa/js-executor/queue/kafkaTemplate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 1fac350976..ad243aacdd 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -93,7 +93,7 @@ function sendMessagesAsBatch() { }).then( () => { logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); - for (let i = 0; i < promisesToSend.length; i++) { + for (let i = 0; i < resolvers.length; i++) { resolvers[i](); } }, From 41391dbef8bbfbf1c0f56165dd41277dc45d23fe Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 1 Jun 2021 18:06:10 +0300 Subject: [PATCH 011/101] js-executor: code format --- .../api/jsInvokeMessageProcessor.js | 56 +++++++++---------- msa/js-executor/queue/kafkaTemplate.js | 46 +++++++-------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 1aabd216d2..bef70936ec 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -43,7 +43,7 @@ function JsInvokeMessageProcessor(producer) { this.lastStatTime = performance.now(); } -JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { +JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function (message) { var tStart = performance.now(); let requestId; let responseTopic; @@ -78,13 +78,13 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var tFinish = performance.now(); var tTook = tFinish - tStart; - if ( tTook > slowQueryLogMs ) { + if (tTook > slowQueryLogMs) { let functionName; if (request.invokeRequest) { try { buf = Buffer.from(request.invokeRequest['functionName']); functionName = buf.toString('utf8'); - } catch (err){ + } catch (err) { logger.error('[%s] Failed to read functionName from message header: %s', requestId, err.message); logger.error(err.stack); } @@ -97,7 +97,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { } -JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, headers, compileRequest) { +JsInvokeMessageProcessor.prototype.processCompileRequest = function (requestId, responseTopic, headers, compileRequest) { var scriptId = getScriptId(compileRequest); logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId); @@ -116,7 +116,7 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r ); } -JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, headers, invokeRequest) { +JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, responseTopic, headers, invokeRequest) { var scriptId = getScriptId(invokeRequest); logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; @@ -160,7 +160,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re ); } -JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) { +JsInvokeMessageProcessor.prototype.processReleaseRequest = function (requestId, responseTopic, headers, releaseRequest) { var scriptId = getScriptId(releaseRequest); logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); if (this.scriptMap.has(scriptId)) { @@ -193,9 +193,9 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT ); } -JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { +JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scriptBody) { var self = this; - return new Promise(function(resolve, reject) { + return new Promise(function (resolve, reject) { if (self.scriptMap.has(scriptId)) { resolve(self.scriptMap.get(scriptId)); } else { @@ -212,7 +212,7 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scrip }); } -JsInvokeMessageProcessor.prototype.cacheScript = function(scriptId, script) { +JsInvokeMessageProcessor.prototype.cacheScript = function (scriptId, script) { if (!this.scriptMap.has(scriptId)) { this.scriptIds.push(scriptId); while (this.scriptIds.length > maxActiveScripts) { @@ -229,40 +229,40 @@ JsInvokeMessageProcessor.prototype.cacheScript = function(scriptId, script) { function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) { const requestIdBits = Utils.UUIDToBits(requestId); return { - requestIdMSB: requestIdBits[0], - requestIdLSB: requestIdBits[1], - compileResponse: compileResponse, - invokeResponse: invokeResponse, - releaseResponse: releaseResponse + requestIdMSB: requestIdBits[0], + requestIdLSB: requestIdBits[1], + compileResponse: compileResponse, + invokeResponse: invokeResponse, + releaseResponse: releaseResponse }; } function createCompileResponse(scriptId, success, errorCode, err) { const scriptIdBits = Utils.UUIDToBits(scriptId); - return { - errorCode: errorCode, - success: success, - errorDetails: parseJsErrorDetails(err), - scriptIdMSB: scriptIdBits[0], - scriptIdLSB: scriptIdBits[1] + return { + errorCode: errorCode, + success: success, + errorDetails: parseJsErrorDetails(err), + scriptIdMSB: scriptIdBits[0], + scriptIdLSB: scriptIdBits[1] }; } function createInvokeResponse(result, success, errorCode, err) { - return { - errorCode: errorCode, - success: success, - errorDetails: parseJsErrorDetails(err), - result: result + return { + errorCode: errorCode, + success: success, + errorDetails: parseJsErrorDetails(err), + result: result }; } function createReleaseResponse(scriptId, success) { const scriptIdBits = Utils.UUIDToBits(scriptId); return { - success: success, - scriptIdMSB: scriptIdBits[0], - scriptIdLSB: scriptIdBits[1] + success: success, + scriptIdMSB: scriptIdBits[0], + scriptIdLSB: scriptIdBits[1] }; } diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index ad243aacdd..7fce429d23 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -45,9 +45,9 @@ function KafkaProducer() { const message = { topic: responseTopic, messages: [{ - key: scriptId, - value: rawResponse, - headers: headers.data + key: scriptId, + value: rawResponse, + headers: headers.data }] }; @@ -70,28 +70,28 @@ function pushMessageToSendLater(message) { } function sendLoopWithLinger() { - if (sendLoopInstance) { + if (sendLoopInstance) { logger.debug("Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); clearInterval(sendLoopInstance); - } else { + } else { logger.debug("Starting new send loop with linger [%s]", linger) - } - sendLoopInstance = setInterval(sendMessagesAsBatch, linger); + } + sendLoopInstance = setInterval(sendMessagesAsBatch, linger); } function sendMessagesAsBatch() { - if (batchMessages.length > 0) { - logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); - const messagesToSend = batchMessages; - const resolvers = batchResolvers; - batchMessages = []; - batchResolvers = []; - producer.sendBatch({ - topicMessages: messagesToSend, - acks: acks, - compression: compressionType - }).then( - () => { + if (batchMessages.length > 0) { + logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); + const messagesToSend = batchMessages; + const resolvers = batchResolvers; + batchMessages = []; + batchResolvers = []; + producer.sendBatch({ + topicMessages: messagesToSend, + acks: acks, + compression: compressionType + }).then( + () => { logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); for (let i = 0; i < resolvers.length; i++) { resolvers[i](); @@ -103,8 +103,8 @@ function sendMessagesAsBatch() { batchMessages = messagesToSend.concat(batchMessages); batchResolvers = resolvers.concat(batchResolvers); //promises will never be rejected. Will retry forever } - ); - } + ); + } } (async () => { @@ -120,8 +120,8 @@ function sendMessagesAsBatch() { let kafkaConfig = { brokers: kafkaBootstrapServers.split(','), - logLevel: logLevel.INFO, - logCreator: KafkaJsWinstonLogCreator + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator }; if (kafkaClientId) { From ab5f1b5b639236fb7528ff9544425e2310c33dbb Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 2 Jun 2021 10:41:24 +0300 Subject: [PATCH 012/101] js-executor: ScriptMap optimize work --- msa/js-executor/api/jsInvokeMessageProcessor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index bef70936ec..aa8f5d2db0 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -196,8 +196,9 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scriptBody) { var self = this; return new Promise(function (resolve, reject) { - if (self.scriptMap.has(scriptId)) { - resolve(self.scriptMap.get(scriptId)); + const script = self.scriptMap.get(scriptId); + if (script !== undefined) { + resolve(script); } else { self.executor.compileScript(scriptBody).then( (script) => { From ff7fa6237f1f342357995c32488f542c984583d9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 2 Jun 2021 11:05:32 +0300 Subject: [PATCH 013/101] js-executor: zero check and code cleanup --- msa/js-executor/api/jsInvokeMessageProcessor.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index aa8f5d2db0..5257a03168 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -121,9 +121,9 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, r logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; if (this.executedScriptsCounter % statFrequency == 0) { - var nowMs = performance.now(); - var msSinceLastStat = nowMs - this.lastStatTime; - var requestsPerSec = statFrequency / msSinceLastStat * 1000; //msSinceLastStat can't be zero in the real world + const nowMs = performance.now(); + const msSinceLastStat = nowMs - this.lastStatTime; + const requestsPerSec = msSinceLastStat == 0 ? statFrequency : statFrequency / msSinceLastStat * 1000; this.lastStatTime = nowMs; logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s]', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec); } @@ -197,13 +197,13 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scri var self = this; return new Promise(function (resolve, reject) { const script = self.scriptMap.get(scriptId); - if (script !== undefined) { + if (script) { resolve(script); } else { self.executor.compileScript(scriptBody).then( - (script) => { - self.cacheScript(scriptId, script); - resolve(script); + (compiledScript) => { + self.cacheScript(scriptId, compiledScript); + resolve(compiledScript); }, (err) => { reject(err); From dd748245706903847ba72f7c6066e97e026ebe28 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 9 Jun 2021 13:30:28 +0300 Subject: [PATCH 014/101] js-executor: Refactoring Kafka executor for use async/await --- msa/js-executor/queue/kafkaTemplate.js | 68 +++++++++++--------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 7fce429d23..b34727c977 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -36,11 +36,10 @@ let producer; const configEntries = []; let batchMessages = []; -let batchResolvers = []; let sendLoopInstance; function KafkaProducer() { - this.send = (responseTopic, scriptId, rawResponse, headers) => { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { logger.debug('Pending queue response, scriptId: [%s]', scriptId); const message = { topic: responseTopic, @@ -51,60 +50,50 @@ function KafkaProducer() { }] }; - return pushMessageToSendLater(message); + await pushMessageToSendLater(message); } } -function pushMessageToSendLater(message) { - let resolver; - const promise = new Promise((resolve, reject) => { - resolver = resolve; - }); +async function pushMessageToSendLater(message) { batchMessages.push(message); - batchResolvers.push(resolver); if (batchMessages.length >= maxBatchSize) { - sendMessagesAsBatch(); - sendLoopWithLinger(); //reset loop function and reschedule new linger + await sendMessagesAsBatch(true); } - return promise; } function sendLoopWithLinger() { if (sendLoopInstance) { - logger.debug("Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); - clearInterval(sendLoopInstance); + clearTimeout(sendLoopInstance); } else { logger.debug("Starting new send loop with linger [%s]", linger) } - sendLoopInstance = setInterval(sendMessagesAsBatch, linger); + sendLoopInstance = setTimeout(sendMessagesAsBatch, linger); } -function sendMessagesAsBatch() { +async function sendMessagesAsBatch(isImmediately) { + if (sendLoopInstance) { + logger.debug("sendMessagesAsBatch: Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); + clearTimeout(sendLoopInstance); + } + sendLoopInstance = null; if (batchMessages.length > 0) { - logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); + logger.debug('sendMessagesAsBatch, length: [%s], %s', batchMessages.length, isImmediately ? 'immediately' : ''); const messagesToSend = batchMessages; - const resolvers = batchResolvers; batchMessages = []; - batchResolvers = []; - producer.sendBatch({ - topicMessages: messagesToSend, - acks: acks, - compression: compressionType - }).then( - () => { - logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); - for (let i = 0; i < resolvers.length; i++) { - resolvers[i](); - } - }, - (err) => { - logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); - logger.error(err.stack); - batchMessages = messagesToSend.concat(batchMessages); - batchResolvers = resolvers.concat(batchResolvers); //promises will never be rejected. Will retry forever - } - ); + try { + await producer.sendBatch({ + topicMessages: messagesToSend, + acks: acks, + compression: compressionType + }) + logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); + } catch(err) { + logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); + logger.error(err.stack); + batchMessages = messagesToSend.concat(batchMessages); + } } + sendLoopWithLinger(); } (async () => { @@ -285,9 +274,8 @@ async function disconnectProducer() { producer = null; try { logger.info('Stopping loop...'); - //TODO: send handle msg - clearInterval(sendLoopInstance); - sendMessagesAsBatch(); + clearTimeout(sendLoopInstance); + await sendMessagesAsBatch(); await _producer.disconnect(); logger.info('Kafka Producer stopped.'); } catch (e) { From a7a7dca3b3eb76166cf8ed93bf764f2c8c5935b2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 8 Jun 2021 20:03:45 +0300 Subject: [PATCH 015/101] js-executor: upgraded libraries to fix vulnerability warning (@google-cloud/pubsub", amqplib) --- msa/js-executor/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 75f009880b..280979afa6 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -13,8 +13,8 @@ }, "dependencies": { "@azure/service-bus": "^1.1.9", - "@google-cloud/pubsub": "^2.5.0", - "amqplib": "^0.6.0", + "@google-cloud/pubsub": "^2.12.0", + "amqplib": "^0.8.0", "aws-sdk": "^2.741.0", "azure-sb": "^0.11.1", "config": "^3.3.1", From 98bea8eaf579f6641bc8a3dfa43639901aca7c91 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 8 Jun 2021 20:14:10 +0300 Subject: [PATCH 016/101] js-executor: http livenessProbe added --- msa/js-executor/api/httpServer.js | 46 +++++++++++++++++++ .../config/custom-environment-variables.yml | 1 + msa/js-executor/config/default.yml | 1 + msa/js-executor/package.json | 1 + msa/js-executor/server.js | 2 + 5 files changed, 51 insertions(+) create mode 100644 msa/js-executor/api/httpServer.js diff --git a/msa/js-executor/api/httpServer.js b/msa/js-executor/api/httpServer.js new file mode 100644 index 0000000000..996638c7c3 --- /dev/null +++ b/msa/js-executor/api/httpServer.js @@ -0,0 +1,46 @@ +/* + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ + +const config = require('config'), + logger = require('../config/logger')._logger('httpServer'), + express = require('express'); + +const httpPort = Number(config.get('http_port')); + +const app = express(); + +app.get('/livenessProbe', async (req, res) => { + const date = new Date(); + const message = { now: date.toISOString() }; + res.send(message); +}) + +app.listen(httpPort, () => logger.info(`Started http endpoint on port ${httpPort}. Please, use /livenessProbe !`)) \ No newline at end of file diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index c5eb15e8b9..7773fdaf0c 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -16,6 +16,7 @@ queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" +http_port: "HTTP_PORT" # /livenessProbe js: response_poll_interval: "REMOTE_JS_RESPONSE_POLL_INTERVAL_MS" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 5a2c47a675..bcefc1c639 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -16,6 +16,7 @@ queue_type: "kafka" request_topic: "js_eval.requests" +http_port: "8888" # /livenessProbe js: response_poll_interval: "25" diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 280979afa6..ba940ee85a 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -18,6 +18,7 @@ "aws-sdk": "^2.741.0", "azure-sb": "^0.11.1", "config": "^3.3.1", + "express": "^4.17.1", "js-yaml": "^3.14.0", "kafkajs": "^1.15.0", "long": "^4.0.0", diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 0c415cc156..7b2e7e59b8 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -51,3 +51,5 @@ switch (serviceType) { process.exit(-1); } +require('./api/httpServer'); + From 05df5e03efbaff6a1a5b48d27dc6327962ecd4b2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Jun 2021 11:26:33 +0300 Subject: [PATCH 017/101] Revert "js-executor: upgraded libraries to fix vulnerability warning (@google-cloud/pubsub", amqplib)" Untested version with breaking changes declared This reverts commit 80f83a86 --- msa/js-executor/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index ba940ee85a..187144da20 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -13,8 +13,8 @@ }, "dependencies": { "@azure/service-bus": "^1.1.9", - "@google-cloud/pubsub": "^2.12.0", - "amqplib": "^0.8.0", + "@google-cloud/pubsub": "^2.5.0", + "amqplib": "^0.6.0", "aws-sdk": "^2.741.0", "azure-sb": "^0.11.1", "config": "^3.3.1", From 3a31b9c5ea7451dce8e5936e196f9fdbbb75eb4c Mon Sep 17 00:00:00 2001 From: Kien Truong Date: Thu, 3 Jun 2021 11:05:02 +0700 Subject: [PATCH 018/101] Fix wrong configuration key for Compression Type Fix #4678 (cherry picked from commit c176ca94aa0f612c3bb3856ae2acf3a399cc74a4) --- msa/js-executor/queue/kafkaTemplate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index b34727c977..7e19f373a8 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -26,7 +26,7 @@ const acks = Number(config.get('kafka.acks')); const maxBatchSize = Number(config.get('kafka.batch_size')); const linger = Number(config.get('kafka.linger_ms')); const requestTimeout = Number(config.get('kafka.requestTimeout')); -const compressionType = (config.get('kafka.requestTimeout') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; +const compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; let kafkaClient; let kafkaAdmin; From 9a4a94621a88dd7e6e329dc1d7767ff75e311e16 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Jun 2021 14:31:51 +0300 Subject: [PATCH 019/101] js-executor: parameter added for Kafka PARTITIONS_CONSUMED_CONCURRENTLY to decrease max latency while scale down replicas --- msa/js-executor/config/custom-environment-variables.yml | 1 + msa/js-executor/config/default.yml | 1 + msa/js-executor/queue/kafkaTemplate.js | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 7773fdaf0c..34903c4541 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -29,6 +29,7 @@ kafka: acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "${TB_KAFKA_BATCH_SIZE:128}" # for producer linger_ms: "${TB_KAFKA_LINGER_MS:1}" # for producer + partitions_consumed_concurrently: "${PARTITIONS_CONSUMED_CONCURRENTLY:1}" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index bcefc1c639..190cf097ed 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -29,6 +29,7 @@ kafka: acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "128" # for producer linger_ms: "1" # for producer + partitions_consumed_concurrently: "1" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "30000" # The default value in kafkajs is: 30000 compression: "gzip" # gzip or uncompressed topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1" diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 7e19f373a8..46ded135e6 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -27,6 +27,7 @@ const maxBatchSize = Number(config.get('kafka.batch_size')); const linger = Number(config.get('kafka.linger_ms')); const requestTimeout = Number(config.get('kafka.requestTimeout')); const compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; +const partitionsConsumedConcurrently = Number(config.get('kafka.partitions_consumed_concurrently')); let kafkaClient; let kafkaAdmin; @@ -197,7 +198,7 @@ async function sendMessagesAsBatch(isImmediately) { logger.info('Started ThingsBoard JavaScript Executor Microservice.'); await consumer.run({ - //partitionsConsumedConcurrently: 1, // Default: 1 + partitionsConsumedConcurrently: partitionsConsumedConcurrently, eachMessage: async ({topic, partition, message}) => { let headers = message.headers; let key = message.key; From 9ffa50f4a528d3f7b99e167e3bd797213ed26314 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Jun 2021 16:05:50 +0300 Subject: [PATCH 020/101] js-executor: fixed env variables mapping --- msa/js-executor/config/custom-environment-variables.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 34903c4541..4185e65835 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -27,9 +27,9 @@ kafka: servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge - batch_size: "${TB_KAFKA_BATCH_SIZE:128}" # for producer - linger_ms: "${TB_KAFKA_LINGER_MS:1}" # for producer - partitions_consumed_concurrently: "${PARTITIONS_CONSUMED_CONCURRENTLY:1}" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency + batch_size: "TB_KAFKA_BATCH_SIZE" # for producer + linger_ms: "TB_KAFKA_LINGER_MS" # for producer + partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" From 43abe750396d0c2b5ce2b8ddb54b86400e2eb96a Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 10 Jun 2021 16:35:52 +0300 Subject: [PATCH 021/101] UI: Fixed not correct show data in table widget --- .../components/widget/lib/alarms-table-widget.component.ts | 4 +++- .../widget/lib/entities-table-widget.component.ts | 4 +++- .../widget/lib/timeseries-table-widget.component.ts | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index bfcfa65bd4..9cc8fc73d5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -260,7 +260,9 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, public onDataUpdated() { this.updateTitle(true); - this.alarmsDatasource.updateAlarms(); + this.ngZone.run(() => { + this.alarmsDatasource.updateAlarms(); + }); } public pageLinkSortDirection(): SortDirection { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index 0091a96d87..5d4b8db851 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -221,7 +221,9 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni public onDataUpdated() { this.updateTitle(true); - this.entityDatasource.dataUpdated(); + this.ngZone.run(() => { + this.entityDatasource.dataUpdated(); + }); } public pageLinkSortDirection(): SortDirection { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index a866a37780..3453222b16 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -193,7 +193,9 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI } public onDataUpdated() { - this.updateCurrentSourceData(); + this.ngZone.run(() => { + this.updateCurrentSourceData(); + }); } private initialize() { @@ -235,7 +237,7 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI } public getTabLabel(source: TimeseriesTableSource){ - if(this.useEntityLabel){ + if (this.useEntityLabel) { return source.datasource.entityLabel || source.datasource.entityName; } else { return source.datasource.entityName; From 9bb4ae8a5d99fa176add95c2fb7e79341c1c1d84 Mon Sep 17 00:00:00 2001 From: Vladyslav Prykhodko Date: Fri, 11 Jun 2021 08:34:47 +0300 Subject: [PATCH 022/101] UI: Add select content in widget --- .../home/components/dashboard/dashboard.component.html | 2 +- .../widget/lib/alarms-table-widget.component.ts | 8 ++++---- .../widget/lib/entities-table-widget.component.ts | 8 ++++---- .../modules/home/components/widget/widget.component.scss | 8 ++++++++ .../modules/home/components/widget/widget.component.ts | 1 - 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html b/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html index 58dea3b6f6..c48030050d 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html @@ -63,7 +63,7 @@
- + Date: Mon, 14 Jun 2021 11:46:54 +0300 Subject: [PATCH 023/101] UI: Refactoring call ngZone in table widgets --- .../lib/alarms-table-widget.component.ts | 23 +++++++++++-------- .../lib/entities-table-widget.component.ts | 18 +++++++-------- .../lib/timeseries-table-widget.component.ts | 15 ++++++------ 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index 9cc8fc73d5..f2730a1026 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -260,9 +260,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, public onDataUpdated() { this.updateTitle(true); - this.ngZone.run(() => { - this.alarmsDatasource.updateAlarms(); - }); + this.alarmsDatasource.updateAlarms(); } public pageLinkSortDirection(): SortDirection { @@ -428,7 +426,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.displayedColumns.push('actions'); } - this.alarmsDatasource = new AlarmsDatasource(this.subscription, latestDataKeys); + this.alarmsDatasource = new AlarmsDatasource(this.subscription, latestDataKeys, this.ngZone); if (this.enableSelection) { this.alarmsDatasource.selectionModeChanged$.subscribe((selectionMode) => { const hideTitlePanel = selectionMode || this.textSearchMode; @@ -959,7 +957,8 @@ class AlarmsDatasource implements DataSource { private appliedSortOrderLabel: string; constructor(private subscription: IWidgetSubscription, - private dataKeys: Array) { + private dataKeys: Array, + private ngZone: NgZone) { } connect(collectionViewer: CollectionViewer): Observable> { @@ -991,6 +990,7 @@ class AlarmsDatasource implements DataSource { updateAlarms() { const subscriptionAlarms = this.subscription.alarms; let alarms = new Array(); + let isEmptySelection = false; subscriptionAlarms.data.forEach((alarmData) => { alarms.push(this.alarmDataToInfo(alarmData)); }); @@ -1003,7 +1003,7 @@ class AlarmsDatasource implements DataSource { const toRemove = this.selection.selected.filter(alarmId => alarmIds.indexOf(alarmId) === -1); this.selection.deselect(...toRemove); if (this.selection.isEmpty()) { - this.onSelectionModeChanged(false); + isEmptySelection = true; } } const alarmsPageData: PageData = { @@ -1012,9 +1012,14 @@ class AlarmsDatasource implements DataSource { totalElements: subscriptionAlarms.totalElements, hasNext: subscriptionAlarms.hasNext }; - this.alarmsSubject.next(alarms); - this.pageDataSubject.next(alarmsPageData); - this.dataLoading = false; + this.ngZone.run(() => { + if (isEmptySelection) { + this.onSelectionModeChanged(false); + } + this.alarmsSubject.next(alarms); + this.pageDataSubject.next(alarmsPageData); + this.dataLoading = false; + }); } private alarmDataToInfo(alarmData: AlarmData): AlarmDataInfo { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index 5d4b8db851..d2959fe553 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -221,9 +221,7 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni public onDataUpdated() { this.updateTitle(true); - this.ngZone.run(() => { - this.entityDatasource.dataUpdated(); - }); + this.entityDatasource.dataUpdated(); } public pageLinkSortDirection(): SortDirection { @@ -417,8 +415,7 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni if (this.actionCellDescriptors.length) { this.displayedColumns.push('actions'); } - this.entityDatasource = new EntityDatasource( - this.translate, dataKeys, this.subscription); + this.entityDatasource = new EntityDatasource(this.translate, dataKeys, this.subscription, this.ngZone); } private editColumnsToDisplay($event: Event) { @@ -691,7 +688,8 @@ class EntityDatasource implements DataSource { constructor( private translate: TranslateService, private dataKeys: Array, - private subscription: IWidgetSubscription + private subscription: IWidgetSubscription, + private ngZone: NgZone ) { } @@ -734,9 +732,11 @@ class EntityDatasource implements DataSource { totalElements: datasourcesPageData.totalElements, hasNext: datasourcesPageData.hasNext }; - this.entitiesSubject.next(entities); - this.pageDataSubject.next(entitiesPageData); - this.dataLoading = false; + this.ngZone.run(() => { + this.entitiesSubject.next(entities); + this.pageDataSubject.next(entitiesPageData); + this.dataLoading = false; + }); } private datasourceToEntityData(datasource: Datasource, data: DatasourceData[]): EntityData { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index 3453222b16..2709a759fb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -193,9 +193,7 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI } public onDataUpdated() { - this.ngZone.run(() => { - this.updateCurrentSourceData(); - }); + this.updateCurrentSourceData(); } private initialize() { @@ -288,7 +286,7 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI if (this.actionCellDescriptors.length) { source.displayedColumns.push('actions'); } - const tsDatasource = new TimeseriesDatasource(source, this.hideEmptyLines, this.dateFormatFilter, this.datePipe); + const tsDatasource = new TimeseriesDatasource(source, this.hideEmptyLines, this.dateFormatFilter, this.datePipe, this.ngZone); tsDatasource.dataUpdated(this.data); this.sources.push(source); } @@ -547,7 +545,8 @@ class TimeseriesDatasource implements DataSource { private source: TimeseriesTableSource, private hideEmptyLines: boolean, private dateFormatFilter: string, - private datePipe: DatePipe + private datePipe: DatePipe, + private ngZone: NgZone ) { this.source.timeseriesDatasource = this; } @@ -570,8 +569,10 @@ class TimeseriesDatasource implements DataSource { catchError(() => of(emptyPageData())), ).subscribe( (pageData) => { - this.rowsSubject.next(pageData.data); - this.pageDataSubject.next(pageData); + this.ngZone.run(() => { + this.rowsSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + }); } ); } From c3ab678c1b6f60702dd23808766c0782147c4e9c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 14 Jun 2021 11:18:23 +0300 Subject: [PATCH 024/101] OtaPackage improvements --- .../controller/OtaPackageController.java | 4 ++-- .../BaseOtaPackageControllerTest.java | 19 +++++++++++-------- .../thingsboard/rest/client/RestClient.java | 4 ++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index 13d39b0b2a..e28c5e37e1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -128,7 +128,7 @@ public class OtaPackageController extends BaseController { @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @RequestMapping(value = "/otaPackage/{otaPackageId}", method = RequestMethod.POST) @ResponseBody - public OtaPackage saveOtaPackageData(@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId, + public OtaPackageInfo saveOtaPackageData(@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId, @RequestParam(required = false) String checksum, @RequestParam(CHECKSUM_ALGORITHM) String checksumAlgorithmStr, @RequestBody MultipartFile file) throws ThingsboardException { @@ -160,7 +160,7 @@ public class OtaPackageController extends BaseController { otaPackage.setContentType(file.getContentType()); otaPackage.setData(ByteBuffer.wrap(bytes)); otaPackage.setDataSize((long) bytes.length); - OtaPackage savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); + OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); logEntityAction(savedOtaPackage.getId(), savedOtaPackage, null, ActionType.UPDATED, null); return savedOtaPackage; } catch (Exception e) { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java index a11d1b624a..6aa341cef6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java @@ -141,10 +141,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackage savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); Assert.assertEquals(FILE_NAME, savedFirmware.getFileName()); Assert.assertEquals(CONTENT_TYPE, savedFirmware.getContentType()); + Assert.assertEquals(CHECKSUM_ALGORITHM, savedFirmware.getChecksumAlgorithm().name()); + Assert.assertEquals(CHECKSUM, savedFirmware.getChecksum()); } @Test @@ -189,11 +191,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackage savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); OtaPackage foundFirmware = doGet("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString(), OtaPackage.class); Assert.assertNotNull(foundFirmware); - Assert.assertEquals(savedFirmware, foundFirmware); + Assert.assertEquals(savedFirmware, new OtaPackageInfo(foundFirmware)); + Assert.assertEquals(DATA, foundFirmware.getData()); } @Test @@ -228,8 +231,8 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes if (i > 100) { MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackage savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); - otaPackages.add(new OtaPackageInfo(savedFirmware)); + OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + otaPackages.add(savedFirmware); } else { otaPackages.add(savedFirmwareInfo); } @@ -271,7 +274,7 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes if (i > 100) { MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackage savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); savedFirmwareInfo = new OtaPackageInfo(savedFirmware); otaPackagesWithData.add(savedFirmwareInfo); } @@ -318,11 +321,11 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes return doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); } - protected OtaPackage savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception { + protected OtaPackageInfo savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception { MockMultipartHttpServletRequestBuilder postRequest = MockMvcRequestBuilders.multipart(urlTemplate, params); postRequest.file(content); setJwtToken(postRequest); - return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackage.class); + return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackageInfo.class); } } diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 14a008634b..b716935b2a 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -2971,7 +2971,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { return restTemplate.postForEntity(baseURL + "/api/otaPackage", otaPackageInfo, OtaPackageInfo.class).getBody(); } - public OtaPackage saveOtaPackageData(OtaPackageId otaPackageId, String checkSum, ChecksumAlgorithm checksumAlgorithm, MultipartFile file) throws Exception { + public OtaPackageInfo saveOtaPackageData(OtaPackageId otaPackageId, String checkSum, ChecksumAlgorithm checksumAlgorithm, MultipartFile file) throws Exception { HttpHeaders header = new HttpHeaders(); header.setContentType(MediaType.MULTIPART_FORM_DATA); @@ -2993,7 +2993,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } return restTemplate.postForEntity( - baseURL + url, requestEntity, OtaPackage.class, params + baseURL + url, requestEntity, OtaPackageInfo.class, params ).getBody(); } From b4566a93fde50c95ca5912ec6f6addf6ba7efa8f Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 14 Jun 2021 15:42:27 +0300 Subject: [PATCH 025/101] Add mobile dashboard to alarm rules --- .../common/data/device/profile/AlarmRule.java | 2 ++ .../rule/engine/profile/AlarmState.java | 21 ++++++++++++------- .../add-device-profile-dialog.component.html | 3 ++- .../profile/alarm/alarm-rule.component.html | 12 +++++++++++ .../profile/alarm/alarm-rule.component.scss | 18 +++++++++++----- .../profile/alarm/alarm-rule.component.ts | 12 ++++++++--- .../profile/device-profile.component.html | 3 ++- .../dashboard-autocomplete.component.html | 2 +- .../dashboard-autocomplete.component.ts | 4 ++++ ui-ngx/src/app/shared/models/device.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 6 +++++- 11 files changed, 65 insertions(+), 19 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java index 09c9f084cb..f578cd15c3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.validation.NoXss; import javax.validation.Valid; @@ -31,5 +32,6 @@ public class AlarmRule implements Serializable { // Advanced @NoXss private String alarmDetails; + private DashboardId dashboardId; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java index 445b9653a2..614cfd4b9c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.device.profile.AlarmConditionSpecType; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -269,16 +270,22 @@ class AlarmState { private JsonNode createDetails(AlarmRuleState ruleState) { JsonNode alarmDetails; String alarmDetailsStr = ruleState.getAlarmRule().getAlarmDetails(); + DashboardId dashboardId = ruleState.getAlarmRule().getDashboardId(); - if (StringUtils.isNotEmpty(alarmDetailsStr)) { - for (var keyFilter : ruleState.getAlarmRule().getCondition().getCondition()) { - EntityKeyValue entityKeyValue = dataSnapshot.getValue(keyFilter.getKey()); - if (entityKeyValue != null) { - alarmDetailsStr = alarmDetailsStr.replaceAll(String.format("\\$\\{%s}", keyFilter.getKey().getKey()), getValueAsString(entityKeyValue)); + if (StringUtils.isNotEmpty(alarmDetailsStr) || dashboardId != null) { + ObjectNode newDetails = JacksonUtil.newObjectNode(); + if (StringUtils.isNotEmpty(alarmDetailsStr)) { + for (var keyFilter : ruleState.getAlarmRule().getCondition().getCondition()) { + EntityKeyValue entityKeyValue = dataSnapshot.getValue(keyFilter.getKey()); + if (entityKeyValue != null) { + alarmDetailsStr = alarmDetailsStr.replaceAll(String.format("\\$\\{%s}", keyFilter.getKey().getKey()), getValueAsString(entityKeyValue)); + } } + newDetails.put("data", alarmDetailsStr); + } + if (dashboardId != null) { + newDetails.put("dashboardId", dashboardId.getId().toString()); } - ObjectNode newDetails = JacksonUtil.newObjectNode(); - newDetails.put("data", alarmDetailsStr); alarmDetails = newDetails; } else if (currentAlarm != null) { alarmDetails = currentAlarm.getDetails(); diff --git a/ui-ngx/src/app/modules/home/components/profile/add-device-profile-dialog.component.html b/ui-ngx/src/app/modules/home/components/profile/add-device-profile-dialog.component.html index 444aa8fb49..c546cc8912 100644 --- a/ui-ngx/src/app/modules/home/components/profile/add-device-profile-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/add-device-profile-dialog.component.html @@ -46,8 +46,9 @@ formControlName="defaultRuleChainId"> +
{{'device-profile.mobile-dashboard-hint' | translate}}
 {{ disabled ? 'visibility' : (alarmRuleFormGroup.get('alarmDetails').value ? 'edit' : 'add') }}
+
+ + {{ ('device-profile.alarm-rule-mobile-dashboard' | translate) + ': ' }} + + +
{{'device-profile.alarm-rule-mobile-dashboard-hint' | translate}}
+ +
diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.scss b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.scss index a0b8a83cd4..d973f261b7 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.scss +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.scss @@ -18,16 +18,24 @@ .row { margin-top: 1em; } - .tb-alarm-rule-details { + .tb-alarm-rule-details, .tb-alarm-rule-dashboard { padding: 4px; - cursor: pointer; - overflow: hidden; - white-space: nowrap; - text-overflow: ellipsis; &.title { opacity: 0.7; overflow: visible; } } + .tb-alarm-rule-details { + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + cursor: pointer; + } + .tb-alarm-rule-dashboard { + &.dashboard { + width: 100%; + max-width: 350px; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.ts b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.ts index d664c5c2a4..76bfa7698e 100644 --- a/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/alarm/alarm-rule.component.ts @@ -34,6 +34,7 @@ import { EditAlarmDetailsDialogData } from '@home/components/profile/alarm/edit-alarm-details-dialog.component'; import { EntityId } from '@shared/models/id/entity-id'; +import { DashboardId } from '@shared/models/id/dashboard-id'; @Component({ selector: 'tb-alarm-rule', @@ -92,7 +93,8 @@ export class AlarmRuleComponent implements ControlValueAccessor, OnInit, Validat this.alarmRuleFormGroup = this.fb.group({ condition: [null, [Validators.required]], schedule: [null], - alarmDetails: [null] + alarmDetails: [null], + dashboardId: [null] }); this.alarmRuleFormGroup.valueChanges.subscribe(() => { this.updateModel(); @@ -110,7 +112,11 @@ export class AlarmRuleComponent implements ControlValueAccessor, OnInit, Validat writeValue(value: AlarmRule): void { this.modelValue = value; - this.alarmRuleFormGroup.reset(this.modelValue || undefined, {emitEvent: false}); + const model = this.modelValue ? { + ...this.modelValue, + dashboardId: this.modelValue.dashboardId?.id + } : null; + this.alarmRuleFormGroup.reset(model || undefined, {emitEvent: false}); } public openEditDetailsDialog($event: Event) { @@ -143,7 +149,7 @@ export class AlarmRuleComponent implements ControlValueAccessor, OnInit, Validat private updateModel() { const value = this.alarmRuleFormGroup.value; if (this.modelValue) { - this.modelValue = {...this.modelValue, ...value}; + this.modelValue = {...this.modelValue, ...value, dashboardId: value.dashboardId ? new DashboardId(value.dashboardId) : null}; this.propagateChange(this.modelValue); } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html index 5fa6c1acdb..83bba2a266 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html @@ -60,8 +60,9 @@ formControlName="defaultRuleChainId"> +
{{'device-profile.mobile-dashboard-hint' | translate}}
 - + Date: Mon, 14 Jun 2021 17:52:53 +0300 Subject: [PATCH 026/101] UI: Add cache cellContent, cellStyle, rowStyle in entity tables widget --- .../entity/entities-table.component.ts | 2 +- .../lib/alarms-table-widget.component.html | 10 +- .../lib/alarms-table-widget.component.ts | 151 +++++++++------- .../lib/entities-table-widget.component.html | 10 +- .../lib/entities-table-widget.component.ts | 158 ++++++++++------- .../timeseries-table-widget.component.html | 18 +- .../lib/timeseries-table-widget.component.ts | 167 ++++++++++-------- 7 files changed, 301 insertions(+), 215 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index df164adda4..84c421c5c1 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -528,7 +528,7 @@ export class EntitiesTableComponent extends PageComponent implements AfterViewIn const col = this.entitiesTableConfig.columns.indexOf(column); const index = row * this.entitiesTableConfig.columns.length + col; let res = this.cellContentCache[index]; - if (!res) { + if (isUndefined(res)) { res = this.domSanitizer.bypassSecurityTrustHtml(column.cellContentFunction(entity, column.key)); this.cellContentCache[index] = res; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.html index 9685dec25b..0db5c6f094 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.html @@ -81,9 +81,9 @@ {{ column.title }} - + @@ -125,8 +125,8 @@ 'mat-selected': alarmsDatasource.isSelected(alarm), 'tb-current-entity': alarmsDatasource.isCurrentAlarm(alarm), 'invisible': alarmsDatasource.dataLoading}" - *matRowDef="let alarm; columns: displayedColumns;" - [ngStyle]="rowStyle(alarm)" + *matRowDef="let alarm; columns: displayedColumns; let row = index" + [ngStyle]="rowStyle(alarm, row)" (click)="onRowClick($event, alarm)"> = []; + private cellStyleCache: Array = []; + private rowStyleCache: Array = []; + private settings: AlarmsTableWidgetSettings; private widgetConfig: WidgetConfig; private subscription: IWidgetSubscription; @@ -261,6 +273,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, public onDataUpdated() { this.updateTitle(true); this.alarmsDatasource.updateAlarms(); + this.clearCache(); } public pageLinkSortDirection(): SortDirection { @@ -484,6 +497,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, if (this.actionCellDescriptors.length) { this.displayedColumns.push('actions'); } + this.clearCache(); } } as DisplayColumnsPanelData }, @@ -609,91 +623,100 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, return widthStyle(columnWidth); } - public rowStyle(alarm: AlarmDataInfo): any { - let style: any = {}; - if (alarm) { - if (this.rowStylesInfo.useRowStyleFunction && this.rowStylesInfo.rowStyleFunction) { + public rowStyle(alarm: AlarmDataInfo, row: number): any { + let res = this.rowStyleCache[row]; + if (!res) { + res = {}; + if (alarm && this.rowStylesInfo.useRowStyleFunction && this.rowStylesInfo.rowStyleFunction) { try { - style = this.rowStylesInfo.rowStyleFunction(alarm, this.ctx); - if (!isObject(style)) { - throw new TypeError(`${style === null ? 'null' : typeof style} instead of style object`); + res = this.rowStylesInfo.rowStyleFunction(alarm, this.ctx); + if (!isObject(res)) { + throw new TypeError(`${res === null ? 'null' : typeof res} instead of style object`); } - if (Array.isArray(style)) { + if (Array.isArray(res)) { throw new TypeError(`Array instead of style object`); } } catch (e) { - style = {}; + res = {}; console.warn(`Row style function in widget '${this.ctx.widgetTitle}' ` + `returns '${e}'. Please check your row style function.`); } - } else { - style = {}; } + this.rowStyleCache[row] = res; } - return style; + return res; } - public cellStyle(alarm: AlarmDataInfo, key: EntityColumn): any { - let style: any = {}; - if (alarm && key) { - const styleInfo = this.stylesInfo[key.def]; - const value = getAlarmValue(alarm, key); - if (styleInfo.useCellStyleFunction && styleInfo.cellStyleFunction) { - try { - style = styleInfo.cellStyleFunction(value, alarm, this.ctx); - if (!isObject(style)) { - throw new TypeError(`${style === null ? 'null' : typeof style} instead of style object`); - } - if (Array.isArray(style)) { - throw new TypeError(`Array instead of style object`); + public cellStyle(alarm: AlarmDataInfo, key: EntityColumn, row: number): any { + const col = this.columns.indexOf(key); + const index = row * this.columns.length + col; + let res = this.cellStyleCache[index]; + if (!res) { + res = {}; + if (alarm && key) { + const styleInfo = this.stylesInfo[key.def]; + const value = getAlarmValue(alarm, key); + if (styleInfo.useCellStyleFunction && styleInfo.cellStyleFunction) { + try { + res = styleInfo.cellStyleFunction(value, alarm, this.ctx); + if (!isObject(res)) { + throw new TypeError(`${res === null ? 'null' : typeof res} instead of style object`); + } + if (Array.isArray(res)) { + throw new TypeError(`Array instead of style object`); + } + } catch (e) { + res = {}; + console.warn(`Cell style function for data key '${key.label}' in widget '${this.ctx.widgetTitle}' ` + + `returns '${e}'. Please check your cell style function.`); } - } catch (e) { - style = {}; - console.warn(`Cell style function for data key '${key.label}' in widget '${this.ctx.widgetTitle}' ` + - `returns '${e}'. Please check your cell style function.`); + } else { + res = this.defaultStyle(key, value); } - } else { - style = this.defaultStyle(key, value); } + this.cellStyleCache[index] = res; } - if (!style.width) { + if (!res.width) { const columnWidth = this.columnWidth[key.def]; - style = {...style, ...widthStyle(columnWidth)}; + res = Object.assign(res, widthStyle(columnWidth)); } - return style; + return res; } - public cellContent(alarm: AlarmDataInfo, key: EntityColumn): SafeHtml { - if (alarm && key) { - const contentInfo = this.contentsInfo[key.def]; - const value = getAlarmValue(alarm, key); - let content = ''; - if (contentInfo.useCellContentFunction && contentInfo.cellContentFunction) { - try { - content = contentInfo.cellContentFunction(value, alarm, this.ctx); - } catch (e) { - content = '' + value; + public cellContent(alarm: AlarmDataInfo, key: EntityColumn, row: number): SafeHtml { + const col = this.columns.indexOf(key); + const index = row * this.columns.length + col; + let res = this.cellContentCache[index]; + if (isUndefined(res)) { + res = ''; + if (alarm && key) { + const contentInfo = this.contentsInfo[key.def]; + const value = getAlarmValue(alarm, key); + let content = ''; + if (contentInfo.useCellContentFunction && contentInfo.cellContentFunction) { + try { + content = contentInfo.cellContentFunction(value, alarm, this.ctx); + } catch (e) { + content = '' + value; + } + } else { + content = this.defaultContent(key, contentInfo, value); } - } else { - content = this.defaultContent(key, contentInfo, value); - } - - if (!isDefined(content)) { - return ''; - } else { - content = this.utils.customTranslation(content, content); - switch (typeof content) { - case 'string': - return this.domSanitizer.bypassSecurityTrustHtml(content); - default: - return content; + if (isDefined(content)) { + content = this.utils.customTranslation(content, content); + switch (typeof content) { + case 'string': + res = this.domSanitizer.bypassSecurityTrustHtml(content); + break; + default: + res = content; + } } } - - } else { - return ''; + this.cellContentCache[index] = res; } + return res; } public onRowClick($event: Event, alarm: AlarmDataInfo) { @@ -936,6 +959,12 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, isSorting(column: EntityColumn): boolean { return column.type === DataKeyType.alarm && column.name.startsWith('details.'); } + + private clearCache() { + this.cellContentCache.length = 0; + this.cellStyleCache.length = 0; + this.rowStyleCache.length = 0; + } } class AlarmsDatasource implements DataSource { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.html index 680ed3ad97..ca93253ddc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.html @@ -42,9 +42,9 @@ matSort [matSortActive]="sortOrderProperty" [matSortDirection]="pageLinkSortDirection()" matSortDisableClear> {{ column.title }} - + @@ -84,8 +84,8 @@ = []; + private cellStyleCache: Array = []; + private rowStyleCache: Array = []; + private settings: EntitiesTableWidgetSettings; private widgetConfig: WidgetConfig; private subscription: IWidgetSubscription; @@ -222,6 +235,7 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni public onDataUpdated() { this.updateTitle(true); this.entityDatasource.dataUpdated(); + this.clearCache(); } public pageLinkSortDirection(): SortDirection { @@ -460,6 +474,7 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni if (this.actionCellDescriptors.length) { this.displayedColumns.push('actions'); } + this.clearCache(); } } as DisplayColumnsPanelData }, @@ -535,91 +550,98 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni return widthStyle(columnWidth); } - public rowStyle(entity: EntityData): any { - let style: any = {}; - if (entity) { - if (this.rowStylesInfo.useRowStyleFunction && this.rowStylesInfo.rowStyleFunction) { + public rowStyle(entity: EntityData, row: number): any { + let res = this.rowStyleCache[row]; + if (!res) { + res = {}; + if (entity && this.rowStylesInfo.useRowStyleFunction && this.rowStylesInfo.rowStyleFunction) { try { - style = this.rowStylesInfo.rowStyleFunction(entity, this.ctx); - if (!isObject(style)) { - throw new TypeError(`${style === null ? 'null' : typeof style} instead of style object`); + res = this.rowStylesInfo.rowStyleFunction(entity, this.ctx); + if (!isObject(res)) { + throw new TypeError(`${res === null ? 'null' : typeof res} instead of style object`); } - if (Array.isArray(style)) { + if (Array.isArray(res)) { throw new TypeError(`Array instead of style object`); } } catch (e) { - style = {}; + res = {}; console.warn(`Row style function in widget '${this.ctx.widgetTitle}' ` + `returns '${e}'. Please check your row style function.`); } - } else { - style = {}; } + this.rowStyleCache[row] = res; } - return style; - } - - public cellStyle(entity: EntityData, key: EntityColumn): any { - let style: any = {}; - if (entity && key) { - const styleInfo = this.stylesInfo[key.def]; - const value = getEntityValue(entity, key); - if (styleInfo.useCellStyleFunction && styleInfo.cellStyleFunction) { - try { - style = styleInfo.cellStyleFunction(value, entity, this.ctx); - if (!isObject(style)) { - throw new TypeError(`${style === null ? 'null' : typeof style} instead of style object`); - } - if (Array.isArray(style)) { - throw new TypeError(`Array instead of style object`); + return res; + } + + public cellStyle(entity: EntityData, key: EntityColumn, row: number): any { + const col = this.columns.indexOf(key); + const index = row * this.columns.length + col; + let res = this.cellStyleCache[index]; + if (!res) { + res = {}; + if (entity && key) { + const styleInfo = this.stylesInfo[key.def]; + const value = getEntityValue(entity, key); + if (styleInfo.useCellStyleFunction && styleInfo.cellStyleFunction) { + try { + res = styleInfo.cellStyleFunction(value, entity, this.ctx); + if (!isObject(res)) { + throw new TypeError(`${res === null ? 'null' : typeof res} instead of style object`); + } + if (Array.isArray(res)) { + throw new TypeError(`Array instead of style object`); + } + } catch (e) { + res = {}; + console.warn(`Cell style function for data key '${key.label}' in widget '${this.ctx.widgetTitle}' ` + + `returns '${e}'. Please check your cell style function.`); } - } catch (e) { - style = {}; - console.warn(`Cell style function for data key '${key.label}' in widget '${this.ctx.widgetTitle}' ` + - `returns '${e}'. Please check your cell style function.`); } - } else { - style = {}; + this.cellStyleCache[index] = res; } } - if (!style.width) { + if (!res.width) { const columnWidth = this.columnWidth[key.def]; - style = {...style, ...widthStyle(columnWidth)}; + res = Object.assign(res, widthStyle(columnWidth)); } - return style; - } - - public cellContent(entity: EntityData, key: EntityColumn): SafeHtml { - if (entity && key) { - const contentInfo = this.contentsInfo[key.def]; - const value = getEntityValue(entity, key); - let content: string; - if (contentInfo.useCellContentFunction && contentInfo.cellContentFunction) { - try { - content = contentInfo.cellContentFunction(value, entity, this.ctx); - } catch (e) { - content = '' + value; + return res; + } + + public cellContent(entity: EntityData, key: EntityColumn, row: number): SafeHtml { + const col = this.columns.indexOf(key); + const index = row * this.columns.length + col; + let res = this.cellContentCache[index]; + if (isUndefined(res)) { + res = ''; + if (entity && key) { + const contentInfo = this.contentsInfo[key.def]; + const value = getEntityValue(entity, key); + let content: string; + if (contentInfo.useCellContentFunction && contentInfo.cellContentFunction) { + try { + content = contentInfo.cellContentFunction(value, entity, this.ctx); + } catch (e) { + content = '' + value; + } + } else { + content = this.defaultContent(key, contentInfo, value); } - } else { - content = this.defaultContent(key, contentInfo, value); - } - - if (!isDefined(content)) { - return ''; - } else { - content = this.utils.customTranslation(content, content); - switch (typeof content) { - case 'string': - return this.domSanitizer.bypassSecurityTrustHtml(content); - default: - return content; + if (isDefined(content)) { + content = this.utils.customTranslation(content, content); + switch (typeof content) { + case 'string': + res = this.domSanitizer.bypassSecurityTrustHtml(content); + break; + default: + res = content; + } } } - - } else { - return ''; + this.cellContentCache[index] = res; } + return res; } private defaultContent(key: EntityColumn, contentInfo: CellContentInfo, value: any): any { @@ -672,6 +694,12 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni } this.ctx.actionsApi.handleWidgetAction($event, actionDescriptor, entityId, entityName, {entity}, entityLabel); } + + private clearCache() { + this.cellContentCache.length = 0; + this.cellStyleCache.length = 0; + this.rowStyleCache.length = 0; + } } class EntityDatasource implements DataSource { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.html index a92b4e16bb..4524082e5b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.html @@ -39,23 +39,23 @@ - +
Timestamp - + {{ h.dataKey.label }} - + @@ -93,8 +93,8 @@ -
= []; + private cellStyleCache: Array = []; + private rowStyleCache: Array = []; + private settings: TimeseriesTableWidgetSettings; private widgetConfig: WidgetConfig; private data: Array; @@ -194,6 +198,7 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI public onDataUpdated() { this.updateCurrentSourceData(); + this.clearCache(); } private initialize() { @@ -337,6 +342,7 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI onSourceIndexChanged() { this.updateCurrentSourceData(); this.updateActiveEntityInfo(); + this.clearCache(); } private enterFilterMode() { @@ -378,6 +384,7 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI source.pageLink.sortOrder.property = sort.active; source.pageLink.sortOrder.direction = Direction[sort.direction.toUpperCase()]; source.timeseriesDatasource.loadRows(); + this.clearCache(); this.ctx.detectChanges(); } @@ -397,94 +404,109 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI return source.datasource.entityId; } - public rowStyle(source: TimeseriesTableSource, row: TimeseriesRow): any { - let style: any = {}; - if (this.rowStylesInfo.useRowStyleFunction && this.rowStylesInfo.rowStyleFunction) { - try { - const rowData = source.rowDataTemplate; - rowData.Timestamp = row[0]; - source.header.forEach((headerInfo) => { - rowData[headerInfo.dataKey.name] = row[headerInfo.index]; - }); - style = this.rowStylesInfo.rowStyleFunction(rowData, this.ctx); - if (!isObject(style)) { - throw new TypeError(`${style === null ? 'null' : typeof style} instead of style object`); - } - if (Array.isArray(style)) { - throw new TypeError(`Array instead of style object`); - } - } catch (e) { - style = {}; - console.warn(`Row style function in widget ` + - `'${this.ctx.widgetConfig.title}' returns '${e}'. Please check your row style function.`); - } - } - return style; - } - - public cellStyle(source: TimeseriesTableSource, index: number, row: TimeseriesRow, value: any): any { - let style: any = {}; - if (index > 0) { - const styleInfo = source.stylesInfo[index - 1]; - if (styleInfo.useCellStyleFunction && styleInfo.cellStyleFunction) { + public rowStyle(source: TimeseriesTableSource, row: TimeseriesRow, index: number): any { + let res = this.rowStyleCache[index]; + if (!res) { + res = {}; + if (this.rowStylesInfo.useRowStyleFunction && this.rowStylesInfo.rowStyleFunction) { try { const rowData = source.rowDataTemplate; rowData.Timestamp = row[0]; source.header.forEach((headerInfo) => { rowData[headerInfo.dataKey.name] = row[headerInfo.index]; }); - style = styleInfo.cellStyleFunction(value, rowData, this.ctx); - if (!isObject(style)) { - throw new TypeError(`${style === null ? 'null' : typeof style} instead of style object`); + res = this.rowStylesInfo.rowStyleFunction(rowData, this.ctx); + if (!isObject(res)) { + throw new TypeError(`${res === null ? 'null' : typeof res} instead of style object`); } - if (Array.isArray(style)) { + if (Array.isArray(res)) { throw new TypeError(`Array instead of style object`); } } catch (e) { - style = {}; - console.warn(`Cell style function for data key '${source.header[index - 1].dataKey.label}' in widget ` + - `'${this.ctx.widgetConfig.title}' returns '${e}'. Please check your cell style function.`); + res = {}; + console.warn(`Row style function in widget ` + + `'${this.ctx.widgetConfig.title}' returns '${e}'. Please check your row style function.`); } } + this.rowStyleCache[index] = res; } - return style; - } - - public cellContent(source: TimeseriesTableSource, index: number, row: TimeseriesRow, value: any): SafeHtml { - if (index === 0) { - return row.formattedTs; - } else { - let content; - const contentInfo = source.contentsInfo[index - 1]; - if (contentInfo.useCellContentFunction && contentInfo.cellContentFunction) { - try { - const rowData = source.rowDataTemplate; - rowData.Timestamp = row[0]; - source.header.forEach((headerInfo) => { - rowData[headerInfo.dataKey.name] = row[headerInfo.index]; - }); - content = contentInfo.cellContentFunction(value, rowData, this.ctx); - } catch (e) { - content = '' + value; + return res; + } + + public cellStyle(source: TimeseriesTableSource, index: number, row: TimeseriesRow, value: any, rowIndex: number): any { + const cacheIndex = rowIndex * (source.header.length + 1) + index; + let res = this.cellStyleCache[cacheIndex]; + if (!res) { + res = {}; + if (index > 0) { + const styleInfo = source.stylesInfo[index - 1]; + if (styleInfo.useCellStyleFunction && styleInfo.cellStyleFunction) { + try { + const rowData = source.rowDataTemplate; + rowData.Timestamp = row[0]; + source.header.forEach((headerInfo) => { + rowData[headerInfo.dataKey.name] = row[headerInfo.index]; + }); + res = styleInfo.cellStyleFunction(value, rowData, this.ctx); + if (!isObject(res)) { + throw new TypeError(`${res === null ? 'null' : typeof res} instead of style object`); + } + if (Array.isArray(res)) { + throw new TypeError(`Array instead of style object`); + } + } catch (e) { + res = {}; + console.warn(`Cell style function for data key '${source.header[index - 1].dataKey.label}' in widget ` + + `'${this.ctx.widgetConfig.title}' returns '${e}'. Please check your cell style function.`); + } } - } else { - const decimals = (contentInfo.decimals || contentInfo.decimals === 0) ? contentInfo.decimals : this.ctx.widgetConfig.decimals; - const units = contentInfo.units || this.ctx.widgetConfig.units; - content = this.ctx.utils.formatValue(value, decimals, units, true); } + this.cellStyleCache[cacheIndex] = res; + } + return res; + } - if (!isDefined(content)) { - return ''; + public cellContent(source: TimeseriesTableSource, index: number, row: TimeseriesRow, value: any, rowIndex: number): SafeHtml { + const cacheIndex = rowIndex * (source.header.length + 1) + index ; + let res = this.cellContentCache[cacheIndex]; + if (isUndefined(res)) { + res = ''; + if (index === 0) { + res = row.formattedTs; } else { - content = this.utils.customTranslation(content, content); - switch (typeof content) { - case 'string': - return this.domSanitizer.bypassSecurityTrustHtml(content); - default: - return content; + let content; + const contentInfo = source.contentsInfo[index - 1]; + if (contentInfo.useCellContentFunction && contentInfo.cellContentFunction) { + try { + const rowData = source.rowDataTemplate; + rowData.Timestamp = row[0]; + source.header.forEach((headerInfo) => { + rowData[headerInfo.dataKey.name] = row[headerInfo.index]; + }); + content = contentInfo.cellContentFunction(value, rowData, this.ctx); + } catch (e) { + content = '' + value; + } + } else { + const decimals = (contentInfo.decimals || contentInfo.decimals === 0) ? contentInfo.decimals : this.ctx.widgetConfig.decimals; + const units = contentInfo.units || this.ctx.widgetConfig.units; + content = this.ctx.utils.formatValue(value, decimals, units, true); + } + + if (isDefined(content)) { + content = this.utils.customTranslation(content, content); + switch (typeof content) { + case 'string': + res = this.domSanitizer.bypassSecurityTrustHtml(content); + break; + default: + res = content; + } } } + this.cellContentCache[cacheIndex] = res; } + return res; } public onRowClick($event: Event, row: TimeseriesRow) { @@ -530,6 +552,13 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI private loadCurrentSourceRow() { this.sources[this.sourceIndex].timeseriesDatasource.loadRows(); + this.clearCache(); + } + + private clearCache() { + this.cellContentCache.length = 0; + this.cellStyleCache.length = 0; + this.rowStyleCache.length = 0; } } From 810c2a7ed3af17887481382a57cb7b06a54a1a6d Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 14 Jun 2021 18:57:02 +0300 Subject: [PATCH 027/101] UI: Fixed bug, not disabled deleting alias used only in alarm widgets --- .../alias/entity-aliases-dialog.component.ts | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.ts b/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.ts index 08119e6ce8..aeb0f9c286 100644 --- a/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/alias/entity-aliases-dialog.component.ts @@ -98,24 +98,17 @@ export class EntityAliasesDialogComponent extends DialogComponent { if (widget.type === widgetType.rpc) { if (widget.config.targetDeviceAliasIds && widget.config.targetDeviceAliasIds.length > 0) { - const targetDeviceAliasId = widget.config.targetDeviceAliasIds[0]; - widgetsTitleList = this.aliasToWidgetsMap[targetDeviceAliasId]; - if (!widgetsTitleList) { - widgetsTitleList = []; - this.aliasToWidgetsMap[targetDeviceAliasId] = widgetsTitleList; - } - widgetsTitleList.push(widget.config.title); + this.addWidgetTitleToWidgetsMap(widget.config.targetDeviceAliasIds[0], widget.config.title); + } + } else if (widget.type === widgetType.alarm) { + if (widget.config.alarmSource) { + this.addWidgetTitleToWidgetsMap(widget.config.alarmSource.entityAliasId, widget.config.title); } } else { const datasources = this.utils.validateDatasources(widget.config.datasources); datasources.forEach((datasource) => { if (datasource.type === DatasourceType.entity && datasource.entityAliasId) { - widgetsTitleList = this.aliasToWidgetsMap[datasource.entityAliasId]; - if (!widgetsTitleList) { - widgetsTitleList = []; - this.aliasToWidgetsMap[datasource.entityAliasId] = widgetsTitleList; - } - widgetsTitleList.push(widget.config.title); + this.addWidgetTitleToWidgetsMap(datasource.entityAliasId, widget.config.title); } }); } @@ -141,6 +134,15 @@ export class EntityAliasesDialogComponent extends DialogComponent = this.aliasToWidgetsMap[aliasId]; + if (!widgetsTitleList) { + widgetsTitleList = []; + this.aliasToWidgetsMap[aliasId] = widgetsTitleList; + } + widgetsTitleList.push(widgetTitle); + } + private createEntityAliasFormControl(aliasId: string, entityAlias: EntityAlias): AbstractControl { const aliasFormControl = this.fb.group({ id: [aliasId], From 095e8913ac5fff1de993486d88998d7b0c3dfd7a Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 14 Jun 2021 19:39:22 +0300 Subject: [PATCH 028/101] UI: Revert code onRowClick function in table --- .../widget/lib/alarms-table-widget.component.ts | 8 ++++---- .../widget/lib/entities-table-widget.component.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index 7c7b006159..da9608e88c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -720,12 +720,12 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, } public onRowClick($event: Event, alarm: AlarmDataInfo) { + if ($event) { + $event.stopPropagation(); + } + this.alarmsDatasource.toggleCurrentAlarm(alarm); const descriptors = this.ctx.actionsApi.getActionDescriptors('rowClick'); if (descriptors.length) { - if ($event) { - $event.stopPropagation(); - } - this.alarmsDatasource.toggleCurrentAlarm(alarm); let entityId; let entityName; if (alarm && alarm.originator) { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index 470697c615..f1e554bb2b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -661,13 +661,13 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni } public onRowClick($event: Event, entity: EntityData, isDouble?: boolean) { + if ($event) { + $event.stopPropagation(); + } + this.entityDatasource.toggleCurrentEntity(entity); const actionSourceId = isDouble ? 'rowDoubleClick' : 'rowClick'; const descriptors = this.ctx.actionsApi.getActionDescriptors(actionSourceId); if (descriptors.length) { - if ($event) { - $event.stopPropagation(); - } - this.entityDatasource.toggleCurrentEntity(entity); let entityId; let entityName; let entityLabel; From 807a543d1c66ff11da387baccc357cb0025e52af Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 15 Jun 2021 16:50:30 +0300 Subject: [PATCH 029/101] UI: Add validation ota package URL; Improvement ota-package-autocomplete; Refactoring update ota-packages in device-component --- .../home/pages/device/device.component.ts | 18 +++++++++++++----- .../pages/ota-update/ota-update.component.html | 2 +- .../pages/ota-update/ota-update.component.ts | 7 ++++++- .../ota-package-autocomplete.component.html | 3 ++- .../ota-package-autocomplete.component.ts | 2 +- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/device/device.component.ts b/ui-ngx/src/app/modules/home/pages/device/device.component.ts index 1ed9f784f1..b55498ed05 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device.component.ts @@ -35,6 +35,7 @@ import { TranslateService } from '@ngx-translate/core'; import { EntityTableConfig } from '@home/models/entity/entities-table-config.models'; import { Subject } from 'rxjs'; import { OtaUpdateType } from '@shared/models/ota-package.models'; +import { distinctUntilChanged } from 'rxjs/operators'; @Component({ selector: 'tb-device', @@ -78,7 +79,7 @@ export class DeviceComponent extends EntityComponent { } buildForm(entity: DeviceInfo): FormGroup { - return this.fb.group( + const form = this.fb.group( { name: [entity ? entity.name : '', [Validators.required]], deviceProfileId: [entity ? entity.deviceProfileId : null, [Validators.required]], @@ -95,6 +96,17 @@ export class DeviceComponent extends EntityComponent { ) } ); + form.get('deviceProfileId').valueChanges.pipe( + distinctUntilChanged((prev, curr) => prev?.id === curr?.id) + ).subscribe(profileId => { + if (profileId && this.isEdit) { + this.entityForm.patchValue({ + firmwareId: null, + softwareId: null + }, {emitEvent: false}); + } + }); + return form; } updateForm(entity: DeviceInfo) { @@ -156,10 +168,6 @@ export class DeviceComponent extends EntityComponent { this.entityForm.markAsDirty(); } } - this.entityForm.patchValue({ - firmwareId: null, - softwareId: null - }); } } } diff --git a/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.html b/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.html index 0f2662a251..6802768303 100644 --- a/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.html +++ b/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.html @@ -149,7 +149,7 @@ - + ota-update.direct-url-required diff --git a/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.ts b/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.ts index 1182e6c40a..0c2d82e171 100644 --- a/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.ts +++ b/ui-ngx/src/app/modules/home/pages/ota-update/ota-update.component.ts @@ -67,7 +67,7 @@ export class OtaUpdateComponent extends EntityComponent implements O this.entityForm.get('file').updateValueAndValidity({emitEvent: false}); } else { this.entityForm.get('file').clearValidators(); - this.entityForm.get('url').setValidators(Validators.required); + this.entityForm.get('url').setValidators([Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]); this.entityForm.get('file').updateValueAndValidity({emitEvent: false}); this.entityForm.get('url').updateValueAndValidity({emitEvent: false}); } @@ -172,6 +172,11 @@ export class OtaUpdateComponent extends EntityComponent implements O } prepareFormValue(formValue: any): any { + if (formValue.resource === 'url') { + delete formValue.file; + } else { + delete formValue.url; + } delete formValue.resource; delete formValue.generateChecksum; return super.prepareFormValue(formValue); diff --git a/ui-ngx/src/app/shared/components/ota-package/ota-package-autocomplete.component.html b/ui-ngx/src/app/shared/components/ota-package/ota-package-autocomplete.component.html index 51c291006d..b29f32a1c6 100644 --- a/ui-ngx/src/app/shared/components/ota-package/ota-package-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/ota-package/ota-package-autocomplete.component.html @@ -21,7 +21,8 @@ formControlName="packageId" (focusin)="onFocus()" [required]="required" - [matAutocomplete]="packageAutocomplete"> + [matAutocomplete]="packageAutocomplete" + [matAutocompleteDisabled]="disabled"> - - {{ 'admin.oauth2.user-info-uri-required' | translate }} - {{ 'admin.oauth2.uri-pattern-error' | translate }} diff --git a/ui-ngx/src/app/modules/home/pages/admin/oauth2-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/oauth2-settings.component.ts index 4a123c642b..bf5dc75023 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/oauth2-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/oauth2-settings.component.ts @@ -311,8 +311,7 @@ export class OAuth2SettingsComponent extends PageComponent implements OnInit, Ha scope: this.fb.array(registration?.scope ? registration.scope : [], OAuth2SettingsComponent.validateScope), jwkSetUri: [registration?.jwkSetUri ? registration.jwkSetUri : '', Validators.pattern(this.URL_REGEXP)], userInfoUri: [registration?.userInfoUri ? registration.userInfoUri : '', - [Validators.required, - Validators.pattern(this.URL_REGEXP)]], + [Validators.pattern(this.URL_REGEXP)]], clientAuthenticationMethod: [ registration?.clientAuthenticationMethod ? registration.clientAuthenticationMethod : ClientAuthenticationMethod.POST, Validators.required], diff --git a/ui-ngx/src/app/shared/models/oauth2.models.ts b/ui-ngx/src/app/shared/models/oauth2.models.ts index b434302a9a..3d2c125299 100644 --- a/ui-ngx/src/app/shared/models/oauth2.models.ts +++ b/ui-ngx/src/app/shared/models/oauth2.models.ts @@ -54,7 +54,8 @@ export const domainSchemaTranslations = new Map( export enum MapperConfigType{ BASIC = 'BASIC', CUSTOM = 'CUSTOM', - GITHUB = 'GITHUB' + GITHUB = 'GITHUB', + APPLE = 'APPLE' } export enum TenantNameStrategy{ From 9717ef5ff512d2d9148b65ebb6f7d00558ece4d5 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Jun 2021 18:52:07 +0300 Subject: [PATCH 033/101] Fix DefaultTenantProfileConfiguration model --- ui-ngx/src/app/shared/models/tenant.model.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 4ce7acb5c5..badc48bea4 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -49,8 +49,10 @@ export interface DefaultTenantProfileConfiguration { maxRuleNodeExecutionsPerMessage: number; maxEmails: number; maxSms: number; + maxCreatedAlarms: number; defaultStorageTtlDays: number; + alarmsTtlDays: number; } export type TenantProfileConfigurations = DefaultTenantProfileConfiguration; @@ -81,7 +83,9 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxRuleNodeExecutionsPerMessage: 0, maxEmails: 0, maxSms: 0, - defaultStorageTtlDays: 0 + maxCreatedAlarms: 0, + defaultStorageTtlDays: 0, + alarmsTtlDays: 0 }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; break; From 08939e23e76605a270972b0ed6046d9fb308d108 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 16 Jun 2021 19:00:45 +0300 Subject: [PATCH 034/101] Refactoring of the Firmware Update --- .../device/data/lwm2m/OtherConfiguration.java | 2 + .../server/LwM2MFirmwareUpdateStrategy.java | 48 ++++ .../server/LwM2MSoftwareUpdateStrategy.java | 48 ++++ .../lwm2m/server/LwM2mServerListener.java | 5 +- .../lwm2m/server/LwM2mSessionMsgListener.java | 3 +- .../server/LwM2mTransportServerHelper.java | 6 +- .../lwm2m/server/LwM2mTransportUtil.java | 215 +++------------ .../lwm2m/server/UpdateResultFw.java | 70 +++++ .../transport/lwm2m/server/UpdateStateFw.java | 56 ++++ .../DefaultLwM2MAttributesService.java | 183 +++++++++---- .../attributes/LwM2MAttributesService.java | 4 +- .../lwm2m/server/client/LwM2mClient.java | 9 +- .../server/client/LwM2mClientContext.java | 5 +- .../server/client/LwM2mClientContextImpl.java | 41 ++- .../lwm2m/server/client/LwM2mFwSwUpdate.java | 65 ++--- .../downlink/TbLwM2MWriteReplaceRequest.java | 6 +- .../ota/DefaultLwM2MOtaUpdateService.java | 248 ++++++++++++++++-- .../lwm2m/server/ota/LwM2MClientOtaInfo.java | 99 +++++++ .../lwm2m/server/ota/LwM2MClientOtaState.java | 5 +- .../server/ota/LwM2MOtaUpdateService.java | 13 + .../rpc/DefaultLwM2MRpcRequestHandler.java | 2 +- .../uplink/DefaultLwM2MUplinkMsgHandler.java | 159 ++--------- .../server/uplink/LwM2mUplinkMsgHandler.java | 4 - 23 files changed, 851 insertions(+), 445 deletions(-) create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MFirmwareUpdateStrategy.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MSoftwareUpdateStrategy.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateStateFw.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/lwm2m/OtherConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/lwm2m/OtherConfiguration.java index 2bfef74391..829180983b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/lwm2m/OtherConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/lwm2m/OtherConfiguration.java @@ -23,5 +23,7 @@ public class OtherConfiguration { private Integer fwUpdateStrategy; private Integer swUpdateStrategy; private Integer clientOnlyObserveAfterConnect; + private String fwUpdateRecourse; + private String swUpdateRecourse; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MFirmwareUpdateStrategy.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MFirmwareUpdateStrategy.java new file mode 100644 index 0000000000..f111152b76 --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MFirmwareUpdateStrategy.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server; + +public enum LwM2MFirmwareUpdateStrategy { + OBJ_5_BINARY(1, "ObjectId 5, Binary"), + OBJ_5_TEMP_URL(2, "ObjectId 5, URI"), + OBJ_19_BINARY(3, "ObjectId 19, Binary"); + + public int code; + public String type; + + LwM2MFirmwareUpdateStrategy(int code, String type) { + this.code = code; + this.type = type; + } + + public static LwM2MFirmwareUpdateStrategy fromStrategyFwByType(String type) { + for (LwM2MFirmwareUpdateStrategy to : LwM2MFirmwareUpdateStrategy.values()) { + if (to.type.equals(type)) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported FW State type : %s", type)); + } + + public static LwM2MFirmwareUpdateStrategy fromStrategyFwByCode(int code) { + for (LwM2MFirmwareUpdateStrategy to : LwM2MFirmwareUpdateStrategy.values()) { + if (to.code == code) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported FW Strategy code : %s", code)); + } +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MSoftwareUpdateStrategy.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MSoftwareUpdateStrategy.java new file mode 100644 index 0000000000..0df3f5ca2d --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MSoftwareUpdateStrategy.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server; + +public enum LwM2MSoftwareUpdateStrategy { + BINARY(1, "ObjectId 9, Binary"), + TEMP_URL(2, "ObjectId 9, URI"); + + public int code; + public String type; + + LwM2MSoftwareUpdateStrategy(int code, String type) { + this.code = code; + this.type = type; + } + + public static LwM2MSoftwareUpdateStrategy fromStrategySwByType(String type) { + for (LwM2MSoftwareUpdateStrategy to : LwM2MSoftwareUpdateStrategy.values()) { + if (to.type.equals(type)) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported SW Strategy type : %s", type)); + } + + public static LwM2MSoftwareUpdateStrategy fromStrategySwByCode(int code) { + for (LwM2MSoftwareUpdateStrategy to : LwM2MSoftwareUpdateStrategy.values()) { + if (to.code == code) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported SW Strategy code : %s", code)); + } + +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java index 36f1e8a903..1c2bb70145 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java @@ -27,8 +27,7 @@ import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandle import java.util.Collection; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertPathFromObjectIdToIdVer; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; @Slf4j public class LwM2mServerListener { @@ -93,7 +92,7 @@ public class LwM2mServerListener { @Override public void onResponse(Observation observation, Registration registration, ObserveResponse response) { if (registration != null) { - service.onUpdateValueAfterReadResponse(registration, convertPathFromObjectIdToIdVer(observation.getPath().toString(), registration), response); + service.onUpdateValueAfterReadResponse(registration, convertObjectIdToVersionedId(observation.getPath().toString(), registration), response); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mSessionMsgListener.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mSessionMsgListener.java index ebf0a48aba..f39ced6dfc 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mSessionMsgListener.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mSessionMsgListener.java @@ -33,7 +33,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseM import org.thingsboard.server.gen.transport.TransportProtos.ToTransportUpdateCredentialsProto; import org.thingsboard.server.transport.lwm2m.server.attributes.LwM2MAttributesService; import org.thingsboard.server.transport.lwm2m.server.rpc.LwM2MRpcRequestHandler; -import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import java.util.Optional; @@ -54,7 +53,7 @@ public class LwM2mSessionMsgListener implements GenericFutureListener>> futures; private final TransportService transportService; + private final LwM2mTransportServerHelper helper; + private final LwM2mClientContext clientContext; + private final LwM2MTransportServerConfig config; + private final LwM2mUplinkMsgHandler uplinkHandler; + private final LwM2mDownlinkMsgHandler downlinkHandler; + private final LwM2MOtaUpdateService otaUpdateService; @Override public ListenableFuture> getSharedAttributes(LwM2mClient client, Collection keys) { @@ -96,55 +116,114 @@ public class DefaultLwM2MAttributesService implements LwM2MAttributesService { * @param msg - */ @Override - public void onAttributeUpdate(TransportProtos.AttributeUpdateNotificationMsg msg, TransportProtos.SessionInfoProto sessionInfo) { -// LwM2mClient lwM2MClient = clientContext.getClientBySessionInfo(sessionInfo); -// if (msg.getSharedUpdatedCount() > 0 && lwM2MClient != null) { -// log.warn("2) OnAttributeUpdate, SharedUpdatedList() [{}]", msg.getSharedUpdatedList()); -// msg.getSharedUpdatedList().forEach(tsKvProto -> { -// String pathName = tsKvProto.getKv().getKey(); -// String pathIdVer = this.getObjectIdByKeyNameFromProfile(sessionInfo, pathName); -// Object valueNew = getValueFromKvProto(tsKvProto.getKv()); -// if ((OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION).equals(pathName) -// && (!valueNew.equals(lwM2MClient.getFwUpdate().getCurrentVersion()))) -// || (OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.TITLE).equals(pathName) -// && (!valueNew.equals(lwM2MClient.getFwUpdate().getCurrentTitle())))) { -// this.getInfoFirmwareUpdate(lwM2MClient, null); -// } else if ((OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.VERSION).equals(pathName) -// && (!valueNew.equals(lwM2MClient.getSwUpdate().getCurrentVersion()))) -// || (OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.TITLE).equals(pathName) -// && (!valueNew.equals(lwM2MClient.getSwUpdate().getCurrentTitle())))) { -// this.getInfoSoftwareUpdate(lwM2MClient, null); -// } -// if (pathIdVer != null) { -// ResourceModel resourceModel = lwM2MClient.getResourceModel(pathIdVer, this.config -// .getModelProvider()); -// if (resourceModel != null && resourceModel.operations.isWritable()) { -// this.updateResourcesValueToClient(lwM2MClient, this.getResourceValueFormatKv(lwM2MClient, pathIdVer), valueNew, pathIdVer); -// } else { -// log.error("Resource path - [{}] value - [{}] is not Writable and cannot be updated", pathIdVer, valueNew); -// String logMsg = String.format("%s: attributeUpdate: Resource path - %s value - %s is not Writable and cannot be updated", -// LOG_LWM2M_ERROR, pathIdVer, valueNew); -// this.logToTelemetry(lwM2MClient, logMsg); -// } -// } else if (!isFwSwWords(pathName)) { -// log.error("Resource name name - [{}] value - [{}] is not present as attribute/telemetry in profile and cannot be updated", pathName, valueNew); -// String logMsg = String.format("%s: attributeUpdate: attribute name - %s value - %s is not present as attribute in profile and cannot be updated", -// LOG_LWM2M_ERROR, pathName, valueNew); -// this.logToTelemetry(lwM2MClient, logMsg); -// } -// -// }); -// } else if (msg.getSharedDeletedCount() > 0 && lwM2MClient != null) { -// msg.getSharedUpdatedList().forEach(tsKvProto -> { -// String pathName = tsKvProto.getKv().getKey(); -// Object valueNew = getValueFromKvProto(tsKvProto.getKv()); -// if (OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION).equals(pathName) && !valueNew.equals(lwM2MClient.getFwUpdate().getCurrentVersion())) { -// lwM2MClient.getFwUpdate().setCurrentVersion((String) valueNew); -// } -// }); -// log.info("[{}] delete [{}] onAttributeUpdate", msg.getSharedDeletedList(), sessionInfo); -// } else if (lwM2MClient == null) { -// log.error("OnAttributeUpdate, lwM2MClient is null"); -// } + public void onAttributesUpdate(TransportProtos.AttributeUpdateNotificationMsg msg, TransportProtos.SessionInfoProto sessionInfo) { + LwM2mClient lwM2MClient = clientContext.getClientBySessionInfo(sessionInfo); + if (msg.getSharedUpdatedCount() > 0 && lwM2MClient != null) { + String newFirmwareTitle = null; + String newFirmwareVersion = null; + String newFirmwareUrl = null; + String newSoftwareTitle = null; + String newSoftwareVersion = null; + List otherAttributes = new ArrayList<>(); + for (TransportProtos.TsKvProto tsKvProto : msg.getSharedUpdatedList()) { + String attrName = tsKvProto.getKv().getKey(); + if (DefaultLwM2MOtaUpdateService.FIRMWARE_TITLE.equals(attrName)) { + newFirmwareTitle = getStrValue(tsKvProto); + } else if (DefaultLwM2MOtaUpdateService.FIRMWARE_VERSION.equals(attrName)) { + newFirmwareVersion = getStrValue(tsKvProto); + } else if (DefaultLwM2MOtaUpdateService.FIRMWARE_URL.equals(attrName)) { + newFirmwareUrl = getStrValue(tsKvProto); + } else if (DefaultLwM2MOtaUpdateService.SOFTWARE_TITLE.equals(attrName)) { + newSoftwareTitle = getStrValue(tsKvProto); + } else if (DefaultLwM2MOtaUpdateService.SOFTWARE_VERSION.equals(attrName)) { + newSoftwareVersion = getStrValue(tsKvProto); + } else { + otherAttributes.add(tsKvProto); + } + } + if (newFirmwareTitle != null || newFirmwareVersion != null) { + otaUpdateService.onTargetFirmwareUpdate(lwM2MClient, newFirmwareTitle, newFirmwareVersion, Optional.ofNullable(newFirmwareUrl)); + } + if (newSoftwareTitle != null || newSoftwareVersion != null) { + otaUpdateService.onTargetSoftwareUpdate(lwM2MClient, newSoftwareTitle, newSoftwareVersion); + } + if (!otherAttributes.isEmpty()) { + onAttributesUpdate(lwM2MClient, otherAttributes); + } + } else if (msg.getSharedDeletedCount() > 0 && lwM2MClient != null) { + msg.getSharedUpdatedList().forEach(tsKvProto -> { + String pathName = tsKvProto.getKv().getKey(); + Object valueNew = getValueFromKvProto(tsKvProto.getKv()); + if (OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION).equals(pathName) && !valueNew.equals(lwM2MClient.getFwUpdate().getCurrentVersion())) { + lwM2MClient.getFwUpdate().setCurrentVersion((String) valueNew); + } + }); + } else if (lwM2MClient == null) { + log.error("OnAttributeUpdate, lwM2MClient is null"); + } + } + + /** + * #1.1 If two names have equal path => last time attribute + * #2.1 if there is a difference in values between the current resource values and the shared attribute values + * => send to client Request Update of value (new value from shared attribute) + * and LwM2MClient.delayedRequests.add(path) + * #2.1 if there is not a difference in values between the current resource values and the shared attribute values + * + */ + @Override + public void onAttributesUpdate(LwM2mClient lwM2MClient, List tsKvProtos) { + log.trace("[{}] onAttributesUpdate [{}]", lwM2MClient.getEndpoint(), tsKvProtos); + tsKvProtos.forEach(tsKvProto -> { + String pathIdVer = clientContext.getObjectIdByKeyNameFromProfile(lwM2MClient, tsKvProto.getKv().getKey()); + if (pathIdVer != null) { + // #1.1 + if (lwM2MClient.getSharedAttributes().containsKey(pathIdVer)) { + if (tsKvProto.getTs() > lwM2MClient.getSharedAttributes().get(pathIdVer).getTs()) { + lwM2MClient.getSharedAttributes().put(pathIdVer, tsKvProto); + } + } else { + lwM2MClient.getSharedAttributes().put(pathIdVer, tsKvProto); + } + } + }); + // #2.1 + lwM2MClient.getSharedAttributes().forEach((pathIdVer, tsKvProto) -> { + this.pushUpdateToClientIfNeeded(lwM2MClient, this.getResourceValueFormatKv(lwM2MClient, pathIdVer), + getValueFromKvProto(tsKvProto.getKv()), pathIdVer); + }); + } + + private void pushUpdateToClientIfNeeded(LwM2mClient lwM2MClient, Object valueOld, Object newValue, String versionedId) { + if (newValue != null && (valueOld == null || !newValue.toString().equals(valueOld.toString()))) { + TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId).value(newValue).timeout(this.config.getTimeout()).build(); + downlinkHandler.sendWriteReplaceRequest(lwM2MClient, request, new TbLwM2MWriteResponseCallback(uplinkHandler, lwM2MClient, versionedId)); + } else { + log.error("Failed update resource [{}] [{}]", versionedId, newValue); + String logMsg = String.format("%s: Failed update resource versionedId - %s value - %s. Value is not changed or bad", + LOG_LWM2M_ERROR, versionedId, newValue); + uplinkHandler.logToTelemetry(lwM2MClient, logMsg); + log.info("Failed update resource [{}] [{}]", versionedId, newValue); + } + } + + /** + * @param pathIdVer - path resource + * @return - value of Resource into format KvProto or null + */ + private Object getResourceValueFormatKv(LwM2mClient lwM2MClient, String pathIdVer) { + LwM2mResource resourceValue = LwM2mTransportUtil.getResourceValueFromLwM2MClient(lwM2MClient, pathIdVer); + if (resourceValue != null) { + ResourceModel.Type currentType = resourceValue.getType(); + ResourceModel.Type expectedType = helper.getResourceModelTypeEqualsKvProtoValueType(currentType, pathIdVer); + return LwM2mValueConverterImpl.getInstance().convertValue(resourceValue.getValue(), currentType, expectedType, + new LwM2mPath(fromVersionedIdToObjectId(pathIdVer))); + } else { + return null; + } + } + + private String getStrValue(TransportProtos.TsKvProto tsKvProto) { + return tsKvProto.getKv().getStringV(); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/LwM2MAttributesService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/LwM2MAttributesService.java index 008788cb33..b9d7fa9e21 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/LwM2MAttributesService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/LwM2MAttributesService.java @@ -28,5 +28,7 @@ public interface LwM2MAttributesService { void onGetAttributesResponse(TransportProtos.GetAttributeResponseMsg getAttributesResponse, TransportProtos.SessionInfoProto sessionInfo); - void onAttributeUpdate(TransportProtos.AttributeUpdateNotificationMsg attributeUpdateNotification, TransportProtos.SessionInfoProto sessionInfo); + void onAttributesUpdate(TransportProtos.AttributeUpdateNotificationMsg attributeUpdateNotification, TransportProtos.SessionInfoProto sessionInfo); + + void onAttributesUpdate(LwM2mClient lwM2MClient, List tsKvProtos); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java index ec187549c8..21d6ec3d48 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java @@ -38,11 +38,9 @@ import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsRes import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.transport.lwm2m.server.LwM2mQueuedRequest; -import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -50,14 +48,13 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.TRANSPORT_DEFAULT_LWM2M_VERSION; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertPathFromObjectIdToIdVer; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.equalsResourceTypeGetSimpleName; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.fromVersionedIdToObjectId; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.getVerFromPathIdVerOrId; @@ -202,7 +199,7 @@ public class LwM2mClient implements Cloneable { } public Object getResourceValue(String pathRezIdVer, String pathRezId) { - String pathRez = pathRezIdVer == null ? convertPathFromObjectIdToIdVer(pathRezId, this.registration) : pathRezIdVer; + String pathRez = pathRezIdVer == null ? convertObjectIdToVersionedId(pathRezId, this.registration) : pathRezIdVer; if (this.resources.get(pathRez) != null) { return this.resources.get(pathRez).getLwM2mResource().getValue(); } @@ -210,7 +207,7 @@ public class LwM2mClient implements Cloneable { } public Object getResourceNameByRezId(String pathRezIdVer, String pathRezId) { - String pathRez = pathRezIdVer == null ? convertPathFromObjectIdToIdVer(pathRezId, this.registration) : pathRezIdVer; + String pathRez = pathRezIdVer == null ? convertObjectIdToVersionedId(pathRezId, this.registration) : pathRezIdVer; if (this.resources.get(pathRez) != null) { return this.resources.get(pathRez).getResourceModel().name; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContext.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContext.java index 0f2f208b2b..006953af58 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContext.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContext.java @@ -53,7 +53,10 @@ public interface LwM2mClientContext { LwM2mClient getClientByDeviceId(UUID deviceId); - void registerClient(Registration registration, ValidateDeviceCredentialsResponse credentials); + String getObjectIdByKeyNameFromProfile(TransportProtos.SessionInfoProto sessionInfo, String keyName); + + String getObjectIdByKeyNameFromProfile(LwM2mClient lwM2mClient, String keyName); + void registerClient(Registration registration, ValidateDeviceCredentialsResponse credentials); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java index b9206ca4df..d60482a0c1 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java @@ -17,6 +17,7 @@ package org.thingsboard.server.transport.lwm2m.server.client; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.eclipse.leshan.core.model.ResourceModel; import org.eclipse.leshan.core.node.LwM2mPath; import org.eclipse.leshan.server.registration.Registration; import org.springframework.stereotype.Service; @@ -25,6 +26,7 @@ import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTrans import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; +import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.secure.TbLwM2MSecurityInfo; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportContext; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; @@ -40,7 +42,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import static org.eclipse.leshan.core.SecurityMode.NO_SEC; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertPathFromObjectIdToIdVer; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.fromVersionedIdToObjectId; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.validateObjectVerFromKey; @Slf4j @Service @@ -49,6 +53,7 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.c public class LwM2mClientContextImpl implements LwM2mClientContext { private final LwM2mTransportContext context; + private final LwM2MTransportServerConfig config; private final TbEditableSecurityStore securityStore; private final Map lwM2mClientsByEndpoint = new ConcurrentHashMap<>(); private final Map lwM2mClientsByRegistrationId = new ConcurrentHashMap<>(); @@ -160,6 +165,28 @@ public class LwM2mClientContextImpl implements LwM2mClientContext { return lwM2mClient; } + /** + * Get path to resource from profile equal keyName + * + * @param sessionInfo - + * @param keyName - + * @return - + */ + @Override + public String getObjectIdByKeyNameFromProfile(TransportProtos.SessionInfoProto sessionInfo, String keyName) { + return getObjectIdByKeyNameFromProfile(getClientBySessionInfo(sessionInfo), keyName); + } + + @Override + public String getObjectIdByKeyNameFromProfile(LwM2mClient lwM2mClient, String keyName) { + Lwm2mDeviceProfileTransportConfiguration profile = getProfile(lwM2mClient.getProfileId()); + + return profile.getObserveAttr().getKeyName().entrySet().stream() + .filter(e -> e.getValue().equals(keyName) && validateResourceInModel(lwM2mClient, e.getKey(), false)).findFirst().orElseThrow( + () -> new IllegalArgumentException(keyName + " is not configured in the device profile!") + ).getKey(); + } + public Registration getRegistration(String registrationId) { return this.lwM2mClientsByRegistrationId.get(registrationId).getRegistration(); } @@ -200,7 +227,7 @@ public class LwM2mClientContextImpl implements LwM2mClientContext { Arrays.stream(client.getRegistration().getObjectLinks()).forEach(link -> { LwM2mPath pathIds = new LwM2mPath(link.getUrl()); if (!pathIds.isRoot()) { - clientObjects.add(convertPathFromObjectIdToIdVer(link.getUrl(), client.getRegistration())); + clientObjects.add(convertObjectIdToVersionedId(link.getUrl(), client.getRegistration())); } }); return (clientObjects.size() > 0) ? clientObjects : null; @@ -211,4 +238,14 @@ public class LwM2mClientContextImpl implements LwM2mClientContext { return lwM2mClientsByRegistrationId.values().stream().filter(e -> deviceId.equals(e.getDeviceId())).findFirst().orElse(null); } + private boolean validateResourceInModel(LwM2mClient lwM2mClient, String pathIdVer, boolean isWritableNotOptional) { + ResourceModel resourceModel = lwM2mClient.getResourceModel(pathIdVer, this.config + .getModelProvider()); + Integer objectId = new LwM2mPath(fromVersionedIdToObjectId(pathIdVer)).getObjectId(); + String objectVer = validateObjectVerFromKey(pathIdVer); + return resourceModel != null && (isWritableNotOptional ? + objectId != null && objectVer != null && objectVer.equals(lwM2mClient.getRegistration().getSupportedVersion(objectId)) && resourceModel.operations.isWritable() : + objectId != null && objectVer != null && objectVer.equals(lwM2mClient.getRegistration().getSupportedVersion(objectId))); + } + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java index b19fa4ac55..b39f6084cd 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java @@ -23,7 +23,10 @@ import org.eclipse.leshan.server.registration.Registration; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.LwM2mOperationType; +import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.downlink.LwM2mDownlinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; @@ -62,9 +65,9 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.F import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_UPDATE_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_ERROR; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL; +import static org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY; +import static org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; +import static org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL; import static org.thingsboard.server.transport.lwm2m.server.LwM2mOperationType.EXECUTE; import static org.thingsboard.server.transport.lwm2m.server.LwM2mOperationType.WRITE_REPLACE; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_INSTALL_ID; @@ -75,8 +78,8 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.S import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_UPDATE; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_UPDATE_STATE_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_VER_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertPathFromObjectIdToIdVer; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.equalsFwSateToFirmwareUpdateStatus; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.toOtaPackageUpdateStatus; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.splitCamelCaseString; @Slf4j @@ -140,8 +143,8 @@ public class LwM2mFwSwUpdate { private void initPathId() { if (FIRMWARE.equals(this.type)) { - this.pathPackageId = LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy ? - FW_PACKAGE_5_ID : LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == this.updateStrategy ? + this.pathPackageId = LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy ? + FW_PACKAGE_5_ID : LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == this.updateStrategy ? FW_PACKAGE_URI_ID : FW_PACKAGE_19_ID; this.pathStateId = FW_STATE_ID; this.pathResultId = FW_RESULT_ID; @@ -185,26 +188,26 @@ public class LwM2mFwSwUpdate { if (this.currentId != null) { this.stateUpdate = OtaPackageUpdateStatus.INITIATED.name(); this.sendLogs(handler, WRITE_REPLACE.name(), LOG_LWM2M_INFO, null); - String targetIdVer = convertPathFromObjectIdToIdVer(this.pathPackageId, this.lwM2MClient.getRegistration()); + String targetIdVer = convertObjectIdToVersionedId(this.pathPackageId, this.lwM2MClient.getRegistration()); String fwMsg = String.format("%s: Start type operation %s paths: %s", LOG_LWM2M_INFO, LwM2mOperationType.FW_UPDATE.name(), this.pathPackageId); handler.logToTelemetry(fwMsg, lwM2MClient.getRegistration().getId()); log.warn("8) Start firmware Update. Send save to: [{}] ver: [{}] path: [{}]", this.lwM2MClient.getDeviceName(), this.currentVersion, targetIdVer); - if (LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy) { + if (LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy) { int chunkSize = 0; int chunk = 0; byte[] firmwareChunk = handler.otaPackageDataCache.get(this.currentId.toString(), chunkSize, chunk); TbLwM2MWriteReplaceRequest downlink = TbLwM2MWriteReplaceRequest.builder().versionedId(targetIdVer).value(firmwareChunk).timeout(handler.config.getTimeout()).build(); request.sendWriteReplaceRequest(lwM2MClient, downlink, new TbLwM2MWriteResponseCallback(handler, lwM2MClient, targetIdVer)); - } else if (LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == this.updateStrategy) { + } else if (LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == this.updateStrategy) { String apiFont = "coap://176.36.143.9:5685"; String uri = apiFont + "/" + FIRMWARE_UPDATE_COAP_RECOURSE + "/" + this.currentId.toString(); log.warn("89) coapUri: [{}]", uri); //TODO: user this.rpcRequest??? TbLwM2MWriteReplaceRequest downlink = TbLwM2MWriteReplaceRequest.builder().versionedId(targetIdVer).value(uri).timeout(handler.config.getTimeout()).build(); request.sendWriteReplaceRequest(lwM2MClient, downlink, new TbLwM2MWriteResponseCallback(handler, lwM2MClient, targetIdVer)); - } else if (LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY.code == this.updateStrategy) { + } else if (LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY.code == this.updateStrategy) { } } else { @@ -269,7 +272,7 @@ public class LwM2mFwSwUpdate { (this.currentTitle != null && pathName != null && this.currentTitle.equals(pathName))) { fwMsg = String.format("%s: The update was interrupted. The device has the same version: %s.", LOG_LWM2M_ERROR, this.currentVersion); - } else if (updateResultFw != null && updateResultFw > LwM2mTransportUtil.UpdateResultFw.UPDATE_SUCCESSFULLY.code) { + } else if (updateResultFw != null && updateResultFw > UpdateResultFw.UPDATE_SUCCESSFULLY.code) { fwMsg = String.format("%s: The update was interrupted. The device has the status UpdateResult: error (%d).", LOG_LWM2M_ERROR, updateResultFw); } @@ -288,7 +291,7 @@ public class LwM2mFwSwUpdate { */ public boolean conditionalFwExecuteStart() { Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return LwM2mTransportUtil.UpdateResultFw.INITIAL.code == updateResult; + return UpdateResultFw.INITIAL.code == updateResult; } /** @@ -297,7 +300,7 @@ public class LwM2mFwSwUpdate { */ public boolean conditionalFwExecuteAfterSuccess() { Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return LwM2mTransportUtil.UpdateResultFw.UPDATE_SUCCESSFULLY.code == updateResult; + return UpdateResultFw.UPDATE_SUCCESSFULLY.code == updateResult; } /** @@ -306,7 +309,7 @@ public class LwM2mFwSwUpdate { */ public boolean conditionalFwExecuteAfterError() { Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return LwM2mTransportUtil.UpdateResultFw.UPDATE_SUCCESSFULLY.code < updateResult; + return UpdateResultFw.UPDATE_SUCCESSFULLY.code < updateResult; } /** @@ -354,7 +357,7 @@ public class LwM2mFwSwUpdate { */ public void finishFwSwUpdate(DefaultLwM2MUplinkMsgHandler handler, boolean success) { Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - String value = FIRMWARE.equals(this.type) ? LwM2mTransportUtil.UpdateResultFw.fromUpdateResultFwByCode(updateResult.intValue()).type : + String value = FIRMWARE.equals(this.type) ? UpdateResultFw.fromUpdateResultFwByCode(updateResult.intValue()).type : LwM2mTransportUtil.UpdateResultSw.fromUpdateResultSwByCode(updateResult.intValue()).type; String key = splitCamelCaseString((String) this.lwM2MClient.getResourceNameByRezId(null, this.pathResultId)); if (success) { @@ -408,18 +411,18 @@ public class LwM2mFwSwUpdate { public void sendReadObserveInfo(LwM2mDownlinkMsgHandler request) { this.infoFwSwUpdate = true; - this.pendingInfoRequestsStart.add(convertPathFromObjectIdToIdVer( + this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( this.pathStateId, this.lwM2MClient.getRegistration())); - this.pendingInfoRequestsStart.add(convertPathFromObjectIdToIdVer( + this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( this.pathResultId, this.lwM2MClient.getRegistration())); - this.pendingInfoRequestsStart.add(convertPathFromObjectIdToIdVer( + this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( FW_3_VER_ID, this.lwM2MClient.getRegistration())); - if (LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy || - LwM2mTransportUtil.LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY.code == this.updateStrategy || + if (LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy || + LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY.code == this.updateStrategy || SOFTWARE.equals(this.type)) { - this.pendingInfoRequestsStart.add(convertPathFromObjectIdToIdVer( + this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( this.pathVerId, this.lwM2MClient.getRegistration())); - this.pendingInfoRequestsStart.add(convertPathFromObjectIdToIdVer( + this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( this.pathNameId, this.lwM2MClient.getRegistration())); } this.pendingInfoRequestsStart.forEach(versionedId -> { @@ -442,7 +445,7 @@ public class LwM2mFwSwUpdate { public void updateStateOta(DefaultLwM2MUplinkMsgHandler handler, LwM2mDownlinkMsgHandler request, Registration registration, String path, int value) { if (OBJ_5_BINARY.code == this.getUpdateStrategy()) { - if ((convertPathFromObjectIdToIdVer(FW_RESULT_ID, registration).equals(path))) { + if ((convertObjectIdToVersionedId(FW_RESULT_ID, registration).equals(path))) { if (DOWNLOADED.name().equals(this.getStateUpdate()) && this.conditionalFwExecuteStart()) { this.executeFwSwWare(handler, request); @@ -455,23 +458,23 @@ public class LwM2mFwSwUpdate { } } } else if (OBJ_5_TEMP_URL.code == this.getUpdateStrategy()) { - if (this.currentId != null && (convertPathFromObjectIdToIdVer(FW_STATE_ID, registration).equals(path))) { - String state = equalsFwSateToFirmwareUpdateStatus(LwM2mTransportUtil.StateFw.fromStateFwByCode(value)).name(); + if (this.currentId != null && (convertObjectIdToVersionedId(FW_STATE_ID, registration).equals(path))) { + String state = toOtaPackageUpdateStatus(UpdateStateFw.fromStateFwByCode(value)).name(); if (StringUtils.isNotEmpty(state) && !FAILED.name().equals(this.stateUpdate) && !state.equals(this.stateUpdate)) { this.stateUpdate = state; this.sendSateOnThingsBoard(handler); } - if (value == LwM2mTransportUtil.StateFw.DOWNLOADED.code) { + if (value == UpdateStateFw.DOWNLOADED.code) { this.executeFwSwWare(handler, request); } handler.firmwareUpdateState.put(lwM2MClient.getEndpoint(), value); } - if ((convertPathFromObjectIdToIdVer(FW_RESULT_ID, registration).equals(path))) { - if (this.currentId != null && value == LwM2mTransportUtil.UpdateResultFw.INITIAL.code) { + if ((convertObjectIdToVersionedId(FW_RESULT_ID, registration).equals(path))) { + if (this.currentId != null && value == UpdateResultFw.INITIAL.code) { this.setStateUpdate(INITIATED.name()); - } else if (this.currentId != null && value == LwM2mTransportUtil.UpdateResultFw.UPDATE_SUCCESSFULLY.code) { + } else if (this.currentId != null && value == UpdateResultFw.UPDATE_SUCCESSFULLY.code) { this.setStateUpdate(UPDATED.name()); - } else if (value > LwM2mTransportUtil.UpdateResultFw.UPDATE_SUCCESSFULLY.code) { + } else if (value > UpdateResultFw.UPDATE_SUCCESSFULLY.code) { this.setStateUpdate(FAILED.name()); } this.sendSateOnThingsBoard(handler); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteReplaceRequest.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteReplaceRequest.java index 007aa1d3f8..f4b1059944 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteReplaceRequest.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteReplaceRequest.java @@ -17,17 +17,21 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import lombok.Builder; import lombok.Getter; +import org.eclipse.leshan.core.request.ContentFormat; import org.eclipse.leshan.core.response.WriteResponse; import org.thingsboard.server.transport.lwm2m.server.LwM2mOperationType; public class TbLwM2MWriteReplaceRequest extends AbstractTbLwM2MTargetedDownlinkRequest { + @Getter + private final ContentFormat contentFormat; @Getter private final Object value; @Builder - private TbLwM2MWriteReplaceRequest(String versionedId, long timeout, Object value) { + private TbLwM2MWriteReplaceRequest(String versionedId, long timeout, ContentFormat contentFormat, Object value) { super(versionedId, timeout); + this.contentFormat = contentFormat; this.value = value; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java index 0b862ff725..68d7d76b73 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java @@ -17,21 +17,50 @@ package org.thingsboard.server.transport.lwm2m.server.ota; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.eclipse.leshan.core.request.ContentFormat; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.cache.ota.OtaPackageDataCache; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageKey; import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus; import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; +import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; +import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper; +import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; +import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; import org.thingsboard.server.transport.lwm2m.server.attributes.LwM2MAttributesService; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; +import org.thingsboard.server.transport.lwm2m.server.downlink.LwM2mDownlinkMsgHandler; +import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MExecuteCallback; +import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MExecuteRequest; +import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplaceRequest; +import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; +import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteUpdateRequest; +import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; +import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import static org.thingsboard.server.common.data.ota.OtaPackageKey.STATE; import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getAttributeKey; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FIRMWARE_UPDATE_COAP_RECOURSE; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; @Slf4j @Service @@ -39,45 +68,228 @@ import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getAttribute @RequiredArgsConstructor public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { + public static final String FIRMWARE_VERSION = getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION); + public static final String FIRMWARE_TITLE = getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.TITLE); + public static final String FIRMWARE_URL = getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.URL); + public static final String SOFTWARE_VERSION = getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.VERSION); + public static final String SOFTWARE_TITLE = getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.TITLE); + public static final String SOFTWARE_URL = getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.URL); + + private static final String FW_PACKAGE_5_ID = "/5/0/0"; + private static final String FW_URL_ID = "/5/0/1"; + private static final String FW_EXECUTE_ID = "/5/0/2"; private static final String FW_NAME_ID = "/5/0/6"; private static final String FW_VER_ID = "/5/0/7"; private static final String SW_NAME_ID = "/9/0/0"; private static final String SW_VER_ID = "/9/0/1"; - private final Map fwStates = new ConcurrentHashMap<>(); - private final Map swStates = new ConcurrentHashMap<>(); + private final Map fwStates = new ConcurrentHashMap<>(); + private final Map swStates = new ConcurrentHashMap<>(); - private final LwM2MAttributesService attributesService; private final TransportService transportService; private final LwM2mClientContext clientContext; + private final LwM2MTransportServerConfig config; + private final LwM2mUplinkMsgHandler uplinkHandler; + private final LwM2mDownlinkMsgHandler downlinkHandler; + private final OtaPackageDataCache otaPackageDataCache; + private final LwM2mTransportServerHelper helper; + private ExecutorService executor; + + @Autowired + @Lazy + private LwM2MAttributesService attributesService; + + @PostConstruct + public void init() { + //TODO: define parallelism in constant + executor = ThingsBoardExecutors.newWorkStealingPool(4, "LwM2M OTA Updates"); + } @Override public void init(LwM2mClient client) { + //TODO: add locks by client fwInfo. //TODO: check that the client supports FW and SW by checking the supported objects in the model. List attributesToFetch = new ArrayList<>(); - if (client.isValidObjectVersion(FW_NAME_ID) || client.isValidObjectVersion(FW_VER_ID)) { - LwM2MClientOtaState fwState = getOrInitFwSate(client); - attributesToFetch.add(getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.TITLE)); - attributesToFetch.add(getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION)); + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.setSupported(client.isValidObjectVersion(FW_NAME_ID) || client.isValidObjectVersion(FW_VER_ID)); + if (fwInfo.isSupported()) { + attributesToFetch.add(FIRMWARE_TITLE); + attributesToFetch.add(FIRMWARE_VERSION); + attributesToFetch.add(FIRMWARE_URL); + } + + if (!attributesToFetch.isEmpty()) { + var future = attributesService.getSharedAttributes(client, attributesToFetch); + DonAsynchron.withCallback(future, attrs -> { + if (fwInfo.isSupported()) { + Optional newFirmwareTitle = getAttributeValue(attrs, FIRMWARE_TITLE); + Optional newFirmwareVersion = getAttributeValue(attrs, FIRMWARE_VERSION); + Optional newFirmwareUrl = getAttributeValue(attrs, FIRMWARE_URL); + if (newFirmwareTitle.isPresent() && newFirmwareVersion.isPresent()) { + onTargetFirmwareUpdate(client, newFirmwareTitle.get(), newFirmwareVersion.get(), newFirmwareUrl); + } + } + }, throwable -> { + if (fwInfo.isSupported()) { + fwInfo.setTargetFetchFailure(true); + } + }, executor); + } + } + + @Override + public void onTargetFirmwareUpdate(LwM2mClient client, String newFirmwareTitle, String newFirmwareVersion, Optional newFirmwareUrl) { + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.updateTarget(newFirmwareTitle, newFirmwareVersion, newFirmwareUrl); + startFirmwareUpdateIfNeeded(client, fwInfo); + } + + @Override + public void onCurrentFirmwareNameUpdate(LwM2mClient client, String name) { + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.setCurrentName(name); + } + + @Override + public void onCurrentFirmwareVersionUpdate(LwM2mClient client, String version) { + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.setCurrentVersion(version); + } + + @Override + public void onCurrentFirmwareStateUpdate(LwM2mClient client, Long state) { + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + UpdateStateFw newState = UpdateStateFw.fromStateFwByCode(state.intValue()); + if (UpdateStateFw.DOWNLOADED.equals(newState)) { + executeFwUpdate(client); + } + fwInfo.setUpdateState(newState); + sendStateUpdateToTelemetry(client, fwInfo, LwM2mTransportUtil.toOtaPackageUpdateStatus(newState)); + } + + @Override + public void onCurrentFirmwareResultUpdate(LwM2mClient client, Long code) { + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + UpdateResultFw result = UpdateResultFw.fromUpdateResultFwByCode(code.intValue()); + sendStateUpdateToTelemetry(client, fwInfo, LwM2mTransportUtil.toOtaPackageUpdateStatus(result)); + if (result.isAgain && fwInfo.getRetryAttempts() <= 2) { + fwInfo.setRetryAttempts(fwInfo.getRetryAttempts() + 1); + startFirmwareUpdateIfNeeded(client, fwInfo); + } else { + fwInfo.setUpdateResult(result); } + } - if (client.isValidObjectVersion(SW_NAME_ID) || client.isValidObjectVersion(SW_VER_ID)) { - LwM2MClientOtaState swState = getOrInitSwSate(client); - attributesToFetch.add(getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.TITLE)); - attributesToFetch.add(getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.VERSION)); + @Override + public void onTargetSoftwareUpdate(LwM2mClient client, String newSoftwareTitle, String newSoftwareVersion) { + + } + + private void startFirmwareUpdateIfNeeded(LwM2mClient client, LwM2MClientOtaInfo fwInfo) { + if (fwInfo.isUpdateRequired()) { + if (StringUtils.isNotEmpty(fwInfo.getTargetUrl())) { + log.info("[{}] Starting update to [{}{}] using URL: {}", client.getEndpoint(), fwInfo.getTargetName(), fwInfo.getTargetVersion(), fwInfo.getTargetUrl()); + startFirmwareUpdateUsingUrl(client, fwInfo.getTargetUrl()); + } else { + startFirmwareUpdateUsingBinary(client, fwInfo); + } } + } + + private void startFirmwareUpdateUsingUrl(LwM2mClient client, String url) { + String targetIdVer = convertObjectIdToVersionedId(FW_URL_ID, client.getRegistration()); + TbLwM2MWriteUpdateRequest request = TbLwM2MWriteUpdateRequest.builder().versionedId(targetIdVer).value(url).timeout(config.getTimeout()).build(); + downlinkHandler.sendWriteUpdateRequest(client, request, new TbLwM2MWriteResponseCallback(uplinkHandler, client, targetIdVer)); + } + + public void startFirmwareUpdateUsingBinary(LwM2mClient client, LwM2MClientOtaInfo fwInfo) { + String versionedId = convertObjectIdToVersionedId(FW_PACKAGE_5_ID, client.getRegistration()); + this.transportService.process(client.getSession(), createOtaPackageRequestMsg(client.getSession(), OtaPackageType.FIRMWARE.name()), + new TransportServiceCallback<>() { + @Override + public void onSuccess(TransportProtos.GetOtaPackageResponseMsg response) { + if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus()) + && response.getType().equals(OtaPackageType.FIRMWARE.name())) { + UUID otaPackageId = new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB()); + var strategy = fwInfo.getStrategy(); + switch (strategy) { + case OBJ_5_BINARY: + byte[] firmwareChunk = otaPackageDataCache.get(otaPackageId.toString(), 0, 0); + TbLwM2MWriteReplaceRequest writeRequest = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId) + .value(firmwareChunk).contentFormat(ContentFormat.OPAQUE) + .timeout(config.getTimeout()).build(); + downlinkHandler.sendWriteReplaceRequest(client, writeRequest, new TbLwM2MWriteResponseCallback(uplinkHandler, client, versionedId)); + break; + case OBJ_5_TEMP_URL: + startFirmwareUpdateUsingUrl(client, fwInfo.getBaseUrl() + "/" + FIRMWARE_UPDATE_COAP_RECOURSE + "/" + otaPackageId.toString()); + break; + default: + //TODO: send log to telemetry + } + } else { + //TODO: send log to telemetry + } + } - var future = attributesService.getSharedAttributes(client, attributesToFetch); + @Override + public void onError(Throwable e) { + log.trace("Failed to process firmwareUpdate ", e); + } + }); } - private LwM2MClientOtaState getOrInitFwSate(LwM2mClient client) { - //TODO: fetch state from the cache. - return fwStates.computeIfAbsent(client.getEndpoint(), endpoint -> new LwM2MClientOtaState()); + private TransportProtos.GetOtaPackageRequestMsg createOtaPackageRequestMsg(TransportProtos.SessionInfoProto sessionInfo, String nameFwSW) { + return TransportProtos.GetOtaPackageRequestMsg.newBuilder() + .setDeviceIdMSB(sessionInfo.getDeviceIdMSB()) + .setDeviceIdLSB(sessionInfo.getDeviceIdLSB()) + .setTenantIdMSB(sessionInfo.getTenantIdMSB()) + .setTenantIdLSB(sessionInfo.getTenantIdLSB()) + .setType(nameFwSW) + .build(); + } + + private void executeFwUpdate(LwM2mClient client) { + TbLwM2MExecuteRequest request = TbLwM2MExecuteRequest.builder().versionedId(FW_EXECUTE_ID).timeout(config.getTimeout()).build(); + downlinkHandler.sendExecuteRequest(client, request, new TbLwM2MExecuteCallback(uplinkHandler, client, FW_EXECUTE_ID)); + } + + private Optional getAttributeValue(List attrs, String keyName) { + for (TransportProtos.TsKvProto attr : attrs) { + if (keyName.equals(attr.getKv().getKey())) { + if (attr.getKv().getType().equals(TransportProtos.KeyValueType.STRING_V)) { + return Optional.of(attr.getKv().getStringV()); + } else { + return Optional.empty(); + } + } + } + return Optional.empty(); + } + + private LwM2MClientOtaInfo getOrInitFwInfo(LwM2mClient client) { + //TODO: fetch state from the cache or DB. + return fwStates.computeIfAbsent(client.getEndpoint(), endpoint -> { + var profile = clientContext.getProfile(client.getProfileId()); + return new LwM2MClientOtaInfo(endpoint, OtaPackageType.FIRMWARE, profile.getClientLwM2mSettings().getFwUpdateStrategy(), + profile.getClientLwM2mSettings().getFwUpdateRecourse()); + }); + } + + private LwM2MClientOtaInfo getOrInitSwInfo(LwM2mClient client) { + //TODO: fetch state from the cache or DB. + return swStates.computeIfAbsent(client.getEndpoint(), endpoint -> { + var profile = clientContext.getProfile(client.getProfileId()); + return new LwM2MClientOtaInfo(endpoint, OtaPackageType.SOFTWARE, profile.getClientLwM2mSettings().getSwUpdateStrategy(), profile.getClientLwM2mSettings().getSwUpdateRecourse()); + }); + } - private LwM2MClientOtaState getOrInitSwSate(LwM2mClient client) { - //TODO: fetch state from the cache. - return swStates.computeIfAbsent(client.getEndpoint(), endpoint -> new LwM2MClientOtaState()); + private void sendStateUpdateToTelemetry(LwM2mClient client, LwM2MClientOtaInfo fwInfo, OtaPackageUpdateStatus status) { + List result = new ArrayList<>(); + TransportProtos.KeyValueProto.Builder kvProto = TransportProtos.KeyValueProto.newBuilder().setKey(getAttributeKey(fwInfo.getType(), STATE)); + kvProto.setType(TransportProtos.KeyValueType.STRING_V).setStringV(status.name()); + result.add(kvProto.build()); + helper.sendParametersOnThingsboardTelemetry(result, client.getSession()); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java new file mode 100644 index 0000000000..26f7abed4f --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.ota; + +import lombok.Data; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; +import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; + +import java.util.Optional; + +@Data +public class LwM2MClientOtaInfo { + + private final String endpoint; + private final OtaPackageType type; + + private String baseUrl; + private boolean supported; + + private boolean targetFetchFailure; + private String targetName; + private String targetVersion; + private String targetUrl; + + private boolean currentFetchFailure; + private String currentName; + private String currentVersion; + + //TODO: use value from device if applicable; + private LwM2MFirmwareUpdateStrategy strategy; + private UpdateStateFw updateState; + private UpdateResultFw updateResult; + + private String failedPackageId; + private int retryAttempts; + + public LwM2MClientOtaInfo(String endpoint, OtaPackageType type, Integer strategyCode, String baseUrl) { + this.endpoint = endpoint; + this.type = type; + this.strategy = LwM2MFirmwareUpdateStrategy.fromStrategyFwByCode(strategyCode); + this.baseUrl = baseUrl; + } + + public void updateTarget(String targetName, String targetVersion, Optional newFirmwareUrl) { + this.targetName = targetName; + this.targetVersion = targetVersion; + this.targetUrl = newFirmwareUrl.orElse(null); + } + + public boolean isUpdateRequired() { + if (StringUtils.isEmpty(targetName) || + StringUtils.isEmpty(targetVersion) || + (StringUtils.isEmpty(currentName) && StringUtils.isEmpty(currentVersion))) { + return false; + } else { + String targetPackageId = getPackageId(targetName, targetVersion); + String currentPackageId = getPackageId(currentName, currentVersion); + if (StringUtils.isNotEmpty(failedPackageId) && failedPackageId.equals(targetPackageId)) { + return false; + } else { + return !targetPackageId.equals(currentPackageId); + } + } + } + + public void setUpdateResult(UpdateResultFw updateResult) { + this.updateResult = updateResult; + switch (updateResult) { + case INITIAL: + break; + case UPDATE_SUCCESSFULLY: + retryAttempts = 0; + break; + default: + failedPackageId = getPackageId(targetName, targetVersion); + break; + } + } + + private static String getPackageId(String name, String version) { + return (StringUtils.isNotEmpty(name) ? name : "") + (StringUtils.isNotEmpty(version) ? version : ""); + } +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaState.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaState.java index 43dd874a09..b4201f3abd 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaState.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaState.java @@ -15,5 +15,8 @@ */ package org.thingsboard.server.transport.lwm2m.server.ota; -public class LwM2MClientOtaState { +public enum LwM2MClientOtaState { + + IDLE, IN_PROGRESS, SUCCESS, FAILED + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java index 9dcbab5d9e..e1bde7df8b 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java @@ -17,8 +17,21 @@ package org.thingsboard.server.transport.lwm2m.server.ota; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import java.util.Optional; + public interface LwM2MOtaUpdateService { void init(LwM2mClient client); + void onTargetFirmwareUpdate(LwM2mClient client, String newFirmwareTitle, String newFirmwareVersion, Optional newFirmwareUrl); + + void onTargetSoftwareUpdate(LwM2mClient client, String newSoftwareTitle, String newSoftwareVersion); + + void onCurrentFirmwareNameUpdate(LwM2mClient client, String name); + + void onCurrentFirmwareVersionUpdate(LwM2mClient client, String version); + + void onCurrentFirmwareStateUpdate(LwM2mClient client, Long state); + + void onCurrentFirmwareResultUpdate(LwM2mClient client, Long result); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java index b2c8f73ac8..533458ce76 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java @@ -238,7 +238,7 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { IdOrKeyRequest requestParams = JacksonUtil.fromString(rpcRequst.getParams(), IdOrKeyRequest.class); String targetId; if (StringUtils.isNotEmpty(requestParams.getKey())) { - targetId = uplinkHandler.getObjectIdByKeyNameFromProfile(client, requestParams.getKey()); + targetId = clientContext.getObjectIdByKeyNameFromProfile(client, requestParams.getKey()); } else if (StringUtils.isNotEmpty(requestParams.getId())) { targetId = requestParams.getId(); } else { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index ec5fbbb6dd..97fe273857 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -68,23 +68,20 @@ import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mFwSwUpdate; import org.thingsboard.server.transport.lwm2m.server.client.ParametersAnalyzeResult; -import org.thingsboard.server.transport.lwm2m.server.client.ResourceValue; import org.thingsboard.server.transport.lwm2m.server.client.ResultsAddKeyValueProto; import org.thingsboard.server.transport.lwm2m.server.downlink.DownlinkRequestCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.LwM2mDownlinkMsgHandler; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MLatchCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MCancelObserveCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MCancelObserveRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MDiscoverCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MDiscoverRequest; +import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MLatchCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MObserveCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MObserveRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MReadCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MReadRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteAttributesCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteAttributesRequest; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplaceRequest; import org.thingsboard.server.transport.lwm2m.server.ota.LwM2MOtaUpdateService; import org.thingsboard.server.transport.lwm2m.server.rpc.LwM2MRpcRequestHandler; import org.thingsboard.server.transport.lwm2m.server.store.TbLwM2MDtlsSessionStore; @@ -109,8 +106,10 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.FAILED; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper.getValueFromKvProto; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_3_VER_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_ID; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_VER_ID; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_NAME_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_RESULT_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_STATE_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_ERROR; @@ -119,9 +118,8 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.L import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_WARN; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertOtaUpdateValueToString; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertPathFromObjectIdToIdVer; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.fromVersionedIdToObjectId; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.validateObjectVerFromKey; @Slf4j @@ -148,9 +146,11 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { public final Map firmwareUpdateState; - public DefaultLwM2MUplinkMsgHandler(TransportService transportService, LwM2MAttributesService attributesService, LwM2MOtaUpdateService otaService, + public DefaultLwM2MUplinkMsgHandler(TransportService transportService, LwM2MTransportServerConfig config, LwM2mTransportServerHelper helper, LwM2mClientContext clientContext, + @Lazy LwM2MOtaUpdateService otaService, + @Lazy LwM2MAttributesService attributesService, @Lazy LwM2MRpcRequestHandler rpcHandler, @Lazy LwM2mDownlinkMsgHandler defaultLwM2MDownlinkMsgHandler, OtaPackageDataCache otaPackageDataCache, @@ -561,21 +561,16 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { private void updateResourcesValue(LwM2mClient lwM2MClient, LwM2mResource lwM2mResource, String path) { Registration registration = lwM2MClient.getRegistration(); if (lwM2MClient.saveResourceValue(path, lwM2mResource, this.config.getModelProvider())) { - /** version != null - * set setClient_fw_info... = value - **/ - if (lwM2MClient.getFwUpdate() != null && lwM2MClient.getFwUpdate().isInfoFwSwUpdate()) { - lwM2MClient.getFwUpdate().initReadValue(this, this.defaultLwM2MDownlinkMsgHandler, path); - } - if (lwM2MClient.getSwUpdate() != null && lwM2MClient.getSwUpdate().isInfoFwSwUpdate()) { - lwM2MClient.getSwUpdate().initReadValue(this, this.defaultLwM2MDownlinkMsgHandler, path); - } - - if ((convertPathFromObjectIdToIdVer(FW_RESULT_ID, registration).equals(path)) || - (convertPathFromObjectIdToIdVer(FW_STATE_ID, registration).equals(path))) { - LwM2mFwSwUpdate fwUpdate = lwM2MClient.getFwUpdate(this, clientContext); - log.warn("93) path: [{}] value: [{}]", path, lwM2mResource.getValue()); - fwUpdate.updateStateOta(this, defaultLwM2MDownlinkMsgHandler, registration, path, ((Long) lwM2mResource.getValue()).intValue()); + if (path.equals(convertObjectIdToVersionedId(FW_NAME_ID, registration))) { + otaService.onCurrentFirmwareNameUpdate(lwM2MClient, (String) lwM2mResource.getValue()); + } else if (path.equals(convertObjectIdToVersionedId(FW_3_VER_ID, registration))) { + otaService.onCurrentFirmwareVersionUpdate(lwM2MClient, (String) lwM2mResource.getValue()); + } else if (path.equals(convertObjectIdToVersionedId(FW_5_VER_ID, registration))) { + otaService.onCurrentFirmwareVersionUpdate(lwM2MClient, (String) lwM2mResource.getValue()); + } else if (path.equals(convertObjectIdToVersionedId(FW_STATE_ID, registration))) { + otaService.onCurrentFirmwareStateUpdate(lwM2MClient, (Long) lwM2mResource.getValue()); + } else if (path.equals(convertObjectIdToVersionedId(FW_RESULT_ID, registration))) { + otaService.onCurrentFirmwareResultUpdate(lwM2MClient, (Long) lwM2mResource.getValue()); } this.updateAttrTelemetry(registration, Collections.singleton(path)); } else { @@ -684,7 +679,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { String resourceName = names.get(pathIdVer); if (resourceName != null && !resourceName.isEmpty()) { try { - LwM2mResource resourceValue = getResourceValueFromLwM2MClient(lwM2MClient, pathIdVer); + LwM2mResource resourceValue = LwM2mTransportUtil.getResourceValueFromLwM2MClient(lwM2MClient, pathIdVer); if (resourceValue != null) { ResourceModel.Type currentType = resourceValue.getType(); ResourceModel.Type expectedType = this.helper.getResourceModelTypeEqualsKvProtoValueType(currentType, pathIdVer); @@ -720,38 +715,6 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { return null; } - /** - * @param pathIdVer - path resource - * @return - value of Resource into format KvProto or null - */ - private Object getResourceValueFormatKv(LwM2mClient lwM2MClient, String pathIdVer) { - LwM2mResource resourceValue = this.getResourceValueFromLwM2MClient(lwM2MClient, pathIdVer); - if (resourceValue != null) { - ResourceModel.Type currentType = resourceValue.getType(); - ResourceModel.Type expectedType = this.helper.getResourceModelTypeEqualsKvProtoValueType(currentType, pathIdVer); - return this.converter.convertValue(resourceValue.getValue(), currentType, expectedType, - new LwM2mPath(fromVersionedIdToObjectId(pathIdVer))); - } else { - return null; - } - } - - /** - * @param lwM2MClient - - * @param path - - * @return - return value of Resource by idPath - */ - private LwM2mResource getResourceValueFromLwM2MClient(LwM2mClient lwM2MClient, String path) { - LwM2mResource lwm2mResourceValue = null; - ResourceValue resourceValue = lwM2MClient.getResources().get(path); - if (resourceValue != null) { - if (new LwM2mPath(fromVersionedIdToObjectId(path)).isResource()) { - lwm2mResourceValue = lwM2MClient.getResources().get(path).getLwM2mResource(); - } - } - return lwm2mResourceValue; - } - @Override public void onWriteResponseOk(LwM2mClient client, String path, WriteRequest request) { if (request.getNode() instanceof LwM2mResource) { @@ -890,19 +853,6 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { } } - private void pushUpdateToClientIfNeeded(LwM2mClient lwM2MClient, Object valueOld, Object newValue, String versionedId) { - if (newValue != null && (valueOld == null || !newValue.toString().equals(valueOld.toString()))) { - TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId).value(newValue).timeout(this.config.getTimeout()).build(); - defaultLwM2MDownlinkMsgHandler.sendWriteReplaceRequest(lwM2MClient, request, new TbLwM2MWriteResponseCallback(this, lwM2MClient, versionedId)); - } else { - log.error("Failed update resource [{}] [{}]", versionedId, newValue); - String logMsg = String.format("%s: Failed update resource versionedId - %s value - %s. Value is not changed or bad", - LOG_LWM2M_ERROR, versionedId, newValue); - this.logToTelemetry(lwM2MClient, logMsg); - log.info("Failed update resource [{}] [{}]", versionedId, newValue); - } - } - /** * @param updateCredentials - Credentials include config only security Client (without config attr/telemetry...) * config attr/telemetry... in profile @@ -912,59 +862,6 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { log.info("[{}] idList [{}] valueList updateCredentials", updateCredentials.getCredentialsIdList(), updateCredentials.getCredentialsValueList()); } - /** - * Get path to resource from profile equal keyName - * - * @param sessionInfo - - * @param keyName - - * @return - - */ - @Override - public String getObjectIdByKeyNameFromProfile(TransportProtos.SessionInfoProto sessionInfo, String keyName) { - return getObjectIdByKeyNameFromProfile(clientContext.getClientBySessionInfo(sessionInfo), keyName); - } - - @Override - public String getObjectIdByKeyNameFromProfile(LwM2mClient lwM2mClient, String keyName) { - Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2mClient.getProfileId()); - - return profile.getObserveAttr().getKeyName().entrySet().stream() - .filter(e -> e.getValue().equals(keyName) && validateResourceInModel(lwM2mClient, e.getKey(), false)).findFirst().orElseThrow( - () -> new IllegalArgumentException(keyName + " is not configured in the device profile!") - ).getKey(); - } - - /** - * #1.1 If two names have equal path => last time attribute - * #2.1 if there is a difference in values between the current resource values and the shared attribute values - * => send to client Request Update of value (new value from shared attribute) - * and LwM2MClient.delayedRequests.add(path) - * #2.1 if there is not a difference in values between the current resource values and the shared attribute values - * - * @param tsKvProtos - */ - public void onAttributesUpdate(LwM2mClient lwM2MClient, List tsKvProtos) { - log.trace("[{}] onAttributesUpdate [{}]", lwM2MClient.getEndpoint(), tsKvProtos); - tsKvProtos.forEach(tsKvProto -> { - String pathIdVer = this.getObjectIdByKeyNameFromProfile(lwM2MClient, tsKvProto.getKv().getKey()); - if (pathIdVer != null) { - // #1.1 - if (lwM2MClient.getSharedAttributes().containsKey(pathIdVer)) { - if (tsKvProto.getTs() > lwM2MClient.getSharedAttributes().get(pathIdVer).getTs()) { - lwM2MClient.getSharedAttributes().put(pathIdVer, tsKvProto); - } - } else { - lwM2MClient.getSharedAttributes().put(pathIdVer, tsKvProto); - } - } - }); - // #2.1 - lwM2MClient.getSharedAttributes().forEach((pathIdVer, tsKvProto) -> { - this.pushUpdateToClientIfNeeded(lwM2MClient, this.getResourceValueFormatKv(lwM2MClient, pathIdVer), - getValueFromKvProto(tsKvProto.getKv()), pathIdVer); - }); - } - /** * @param lwM2MClient - * @return SessionInfoProto - @@ -1017,7 +914,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { keysToFetch.removeAll(OtaPackageUtil.ALL_FW_ATTRIBUTE_KEYS); keysToFetch.removeAll(OtaPackageUtil.ALL_SW_ATTRIBUTE_KEYS); DonAsynchron.withCallback(attributesService.getSharedAttributes(lwM2MClient, keysToFetch), - v -> onAttributesUpdate(lwM2MClient, v), + v -> attributesService.onAttributesUpdate(lwM2MClient, v), t -> log.error("[{}] Failed to get attributes", lwM2MClient.getEndpoint(), t), registrationExecutor); } @@ -1045,7 +942,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { fwUpdate.setCurrentTitle(response.getTitle()); fwUpdate.setCurrentId(new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB())); // if (rpcRequest == null) { - fwUpdate.sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); + fwUpdate.sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); // } else { // fwUpdate.writeFwSwWare(handler, defaultLwM2MDownlinkMsgHandler); // } @@ -1059,7 +956,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { lwM2MClient.getDeviceName(), response.getResponseStatus().toString()); log.trace(msgError); // if (rpcRequest != null) { - //TODO: refactor + //TODO: refactor // sendErrorRpcResponse(rpcRequest, msgError, sessionInfo); // } } @@ -1091,7 +988,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { lwM2MClient.getSwUpdate().setCurrentId(new OtaPackageId(new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB())).getId()); lwM2MClient.getSwUpdate().sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); // if (rpcRequest == null) { - lwM2MClient.getSwUpdate().sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); + lwM2MClient.getSwUpdate().sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); // } else { // lwM2MClient.getSwUpdate().writeFwSwWare(handler, defaultLwM2MDownlinkMsgHandler); // } @@ -1124,16 +1021,6 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { return profile.getObserveAttr().getKeyName(); } - private boolean validateResourceInModel(LwM2mClient lwM2mClient, String pathIdVer, boolean isWritableNotOptional) { - ResourceModel resourceModel = lwM2mClient.getResourceModel(pathIdVer, this.config - .getModelProvider()); - Integer objectId = new LwM2mPath(fromVersionedIdToObjectId(pathIdVer)).getObjectId(); - String objectVer = validateObjectVerFromKey(pathIdVer); - return resourceModel != null && (isWritableNotOptional ? - objectId != null && objectVer != null && objectVer.equals(lwM2mClient.getRegistration().getSupportedVersion(objectId)) && resourceModel.operations.isWritable() : - objectId != null && objectVer != null && objectVer.equals(lwM2mClient.getRegistration().getSupportedVersion(objectId))); - } - public LwM2MTransportServerConfig getConfig() { return this.config; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java index 041af97474..517db388e4 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java @@ -60,9 +60,5 @@ public interface LwM2mUplinkMsgHandler { void onToTransportUpdateCredentials(TransportProtos.ToTransportUpdateCredentialsProto updateCredentials); - String getObjectIdByKeyNameFromProfile(TransportProtos.SessionInfoProto sessionInfo, String name); - - String getObjectIdByKeyNameFromProfile(LwM2mClient lwM2mClient, String keyName); - LwM2MTransportServerConfig getConfig(); } From 079ae5fd8f17bb7b6f788372e7734bc5e03a8796 Mon Sep 17 00:00:00 2001 From: Vladyslav Prykhodko Date: Wed, 16 Jun 2021 23:06:52 +0300 Subject: [PATCH 035/101] UI: Refactoring resource: add help link; improvement text and view --- .../resources-library-table-config.resolve.ts | 10 +++---- .../resource/resources-library.component.html | 24 ++++++++++++++--- .../resource/resources-library.component.ts | 27 ++++++++++++++----- ui-ngx/src/app/shared/models/constants.ts | 1 + .../src/app/shared/models/resource.models.ts | 4 +-- .../assets/locale/locale.constant-en_US.json | 7 +++-- .../assets/locale/locale.constant-zh_CN.json | 1 - 7 files changed, 54 insertions(+), 20 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts index 089d4b3d82..96745fa47a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts @@ -69,10 +69,10 @@ export class ResourcesLibraryTableConfigResolver implements Resolve true, - onAction: ($event, entity) => this.exportResource($event, entity) + onAction: ($event, entity) => this.downloadResource($event, entity) } ); @@ -118,7 +118,7 @@ export class ResourcesLibraryTableConfigResolver implements Resolve): boolean { switch (action.action) { - case 'uploadResource': - this.exportResource(action.event, action.entity); + case 'downloadResource': + this.downloadResource(action.event, action.entity); return true; } return false; diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html index 5b98fa9c61..1f33d63776 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html @@ -18,16 +18,26 @@
+
+ +
@@ -47,7 +57,7 @@ {{ 'resource.title-required' | translate }} - +
+ + resource.file-name + + +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts index ce26499102..50d565ccc0 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts @@ -30,6 +30,7 @@ import { ResourceTypeTranslationMap } from '@shared/models/resource.models'; import { pairwise, startWith, takeUntil } from 'rxjs/operators'; +import { ActionNotificationShow } from "@core/notification/notification.actions"; @Component({ selector: 'tb-resources-library', @@ -88,26 +89,29 @@ export class ResourcesLibraryComponent extends EntityComponent impleme } buildForm(entity: Resource): FormGroup { - return this.fb.group( + const form = this.fb.group( { + title: [entity ? entity.title : '', []], resourceType: [{ value: entity?.resourceType ? entity.resourceType : ResourceType.LWM2M_MODEL, - disabled: this.isEdit + disabled: !this.isAdd }, [Validators.required]], - data: [entity ? entity.data : null, [Validators.required]], fileName: [entity ? entity.fileName : null, [Validators.required]], - title: [entity ? entity.title : '', []] } ); + if (this.isAdd) { + form.addControl('data', this.fb.control(null, Validators.required)); + } + return form } updateForm(entity: Resource) { - this.entityForm.patchValue({resourceType: entity.resourceType}); if (this.isEdit) { this.entityForm.get('resourceType').disable({emitEvent: false}); + this.entityForm.get('fileName').disable({emitEvent: false}); } this.entityForm.patchValue({ - data: entity.data, + resourceType: entity.resourceType, fileName: entity.fileName, title: entity.title }); @@ -132,4 +136,15 @@ export class ResourcesLibraryComponent extends EntityComponent impleme convertToBase64File(data: string): string { return window.btoa(data); } + + onResourceIdCopied() { + this.store.dispatch(new ActionNotificationShow( + { + message: this.translate.instant('resource.idCopiedMessage'), + type: 'success', + duration: 750, + verticalPosition: 'bottom', + horizontalPosition: 'right' + })); + } } diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index b72e2b196d..56347f1261 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -120,6 +120,7 @@ export const HelpLinks = { entityViews: helpBaseUrl + '/docs/user-guide/ui/entity-views', entitiesImport: helpBaseUrl + '/docs/user-guide/bulk-provisioning', rulechains: helpBaseUrl + '/docs/user-guide/ui/rule-chains', + resources: helpBaseUrl + '/docs/user-guide/ui/resources', dashboards: helpBaseUrl + '/docs/user-guide/ui/dashboards', otaUpdates: helpBaseUrl + '/docs/user-guide/ui/ota-updates', widgetsBundles: helpBaseUrl + '/docs/user-guide/ui/widget-library#bundles', diff --git a/ui-ngx/src/app/shared/models/resource.models.ts b/ui-ngx/src/app/shared/models/resource.models.ts index 082d09ac24..9c478cca0d 100644 --- a/ui-ngx/src/app/shared/models/resource.models.ts +++ b/ui-ngx/src/app/shared/models/resource.models.ts @@ -61,6 +61,6 @@ export interface Resource extends ResourceInfo { } export interface Resources extends ResourceInfo { - data: string|string[]; - fileName: string|string[]; + data: Array; + fileName: Array; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ec084b8a3a..172cc2033d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2309,20 +2309,23 @@ }, "resource": { "add": "Add Resource", + "copyId": "Copy resource Id", "delete": "Delete resource", "delete-resource-text": "Be careful, after the confirmation the resource will become unrecoverable.", "delete-resource-title": "Are you sure you want to delete the resource '{{resourceTitle}}'?", "delete-resources-action-title": "Delete { count, plural, 1 {1 resource} other {# resources} }", "delete-resources-text": "Be careful, after the confirmation all selected resources will be removed.", "delete-resources-title": "Are you sure you want to delete { count, plural, 1 {1 resource} other {# resources} }?", + "download": "Download resource", "drop-file": "Drop a resource file or click to select a file to upload.", "empty": "Resource is empty", - "export": "Export resource", + "file-name": "File name", + "idCopiedMessage": "Resource Id has been copied to clipboard", "no-resource-matching": "No resource matching '{{widgetsBundle}}' were found.", "no-resource-text": "No resources found", "open-widgets-bundle": "Open widgets bundle", "resource": "Resource", - "resource-library-details": "Resource library details", + "resource-library-details": "Resource details", "resource-type": "Resource type", "resources-library": "Resources library", "search": "Search resources", diff --git a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json index 6a60db96fc..cc23f5fc78 100644 --- a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json +++ b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json @@ -1994,7 +1994,6 @@ "delete-resources-title": "确定要删除 { count, plural, 1 {# 个资源} other {# 个资源} }?", "drop-file": "拖拽资源文件或单击以选择要上传的文件。", "empty": "资源为空", - "export": "导出资源", "no-resource-matching": "找不到与 '{{widgetsBundle}}' 匹配的资源。", "no-resource-text": "找不到资源", "open-widgets-bundle": "打开部件库", From cda6f3678b8cfe7a0046d75ff5daeae3287b8312 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 07:27:42 +0300 Subject: [PATCH 036/101] js-executor: fixed license header --- msa/js-executor/api/httpServer.js | 36 +++++++++---------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/msa/js-executor/api/httpServer.js b/msa/js-executor/api/httpServer.js index 996638c7c3..4948c45f61 100644 --- a/msa/js-executor/api/httpServer.js +++ b/msa/js-executor/api/httpServer.js @@ -1,34 +1,18 @@ /* - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - const config = require('config'), logger = require('../config/logger')._logger('httpServer'), express = require('express'); From c07379812b213f3ca249da31285a00124d6345bd Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 07:28:53 +0300 Subject: [PATCH 037/101] js-executor: updated yarn.lock --- msa/js-executor/yarn.lock | 361 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 347 insertions(+), 14 deletions(-) diff --git a/msa/js-executor/yarn.lock b/msa/js-executor/yarn.lock index 404c059c23..0092372b69 100644 --- a/msa/js-executor/yarn.lock +++ b/msa/js-executor/yarn.lock @@ -418,6 +418,14 @@ abort-controller@^3.0.0: dependencies: event-target-shim "^5.0.0" +accepts@~1.3.7: + version "1.3.7" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" + integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== + dependencies: + mime-types "~2.1.24" + negotiator "0.6.2" + agent-base@6: version "6.0.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.1.tgz#808007e4e5867decb0ab6ab2f928fbdb5a596db4" @@ -487,6 +495,11 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" +array-flatten@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" + integrity sha1-ml9pkFGx5wczKPKgCJaLZOopVdI= + array-union@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" @@ -621,6 +634,22 @@ bluebird@^3.5.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +body-parser@1.19.0: + version "1.19.0" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a" + integrity sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw== + dependencies: + bytes "3.1.0" + content-type "~1.0.4" + debug "2.6.9" + depd "~1.1.2" + http-errors "1.7.2" + iconv-lite "0.4.24" + on-finished "~2.3.0" + qs "6.7.0" + raw-body "2.4.0" + type-is "~1.6.17" + boxen@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" @@ -682,6 +711,11 @@ byline@^5.0.0: resolved "https://registry.yarnpkg.com/byline/-/byline-5.0.0.tgz#741c5216468eadc457b03410118ad77de8c1ddb1" integrity sha1-dBxSFkaOrcRXsDQQEYrXfejB3bE= +bytes@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" + integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== + cacheable-request@^6.0.0: version "6.1.0" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" @@ -838,6 +872,28 @@ configstore@^5.0.1: write-file-atomic "^3.0.0" xdg-basedir "^4.0.0" +content-disposition@0.5.3: + version "0.5.3" + resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" + integrity sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g== + dependencies: + safe-buffer "5.1.2" + +content-type@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" + integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== + +cookie-signature@1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" + integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= + +cookie@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba" + integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg== + core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -867,6 +923,13 @@ dateformat@1.0.2-1.2.3: dependencies: ms "^2.1.1" +debug@2.6.9, debug@^2.2.0, debug@~2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + debug@4, debug@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" @@ -874,13 +937,6 @@ debug@4, debug@^4.1.1: dependencies: ms "^2.1.1" -debug@^2.2.0, debug@~2.6.9: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== - dependencies: - ms "2.0.0" - decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -913,6 +969,16 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= +depd@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" + integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= + +destroy@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" + integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -962,6 +1028,11 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: dependencies: safe-buffer "^5.0.1" +ee-first@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" + integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= + emoji-regex@^7.0.1: version "7.0.3" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" @@ -977,6 +1048,11 @@ enabled@2.0.x: resolved "https://registry.yarnpkg.com/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2" integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== +encodeurl@~1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" + integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= + end-of-stream@^1.0.0, end-of-stream@^1.1.0: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -994,6 +1070,11 @@ escape-goat@^2.0.0: resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" integrity sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q== +escape-html@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" + integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg= + escodegen@^1.14.1: version "1.14.3" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" @@ -1021,6 +1102,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +etag@~1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" + integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= + event-target-shim@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" @@ -1041,6 +1127,42 @@ expand-template@^2.0.3: resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== +express@^4.17.1: + version "4.17.1" + resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" + integrity sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g== + dependencies: + accepts "~1.3.7" + array-flatten "1.1.1" + body-parser "1.19.0" + content-disposition "0.5.3" + content-type "~1.0.4" + cookie "0.4.0" + cookie-signature "1.0.6" + debug "2.6.9" + depd "~1.1.2" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + finalhandler "~1.1.2" + fresh "0.5.2" + merge-descriptors "1.0.1" + methods "~1.1.2" + on-finished "~2.3.0" + parseurl "~1.3.3" + path-to-regexp "0.1.7" + proxy-addr "~2.0.5" + qs "6.7.0" + range-parser "~1.2.1" + safe-buffer "5.1.2" + send "0.17.1" + serve-static "1.14.1" + setprototypeof "1.1.1" + statuses "~1.5.0" + type-is "~1.6.18" + utils-merge "1.0.1" + vary "~1.1.2" + extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -1119,6 +1241,19 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +finalhandler@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" + integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== + dependencies: + debug "2.6.9" + encodeurl "~1.0.2" + escape-html "~1.0.3" + on-finished "~2.3.0" + parseurl "~1.3.3" + statuses "~1.5.0" + unpipe "~1.0.0" + find-up@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" @@ -1155,6 +1290,16 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +forwarded@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" + integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== + +fresh@0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" + integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= + from2@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af" @@ -1384,6 +1529,28 @@ http-cache-semantics@^4.0.0: resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== +http-errors@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f" + integrity sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg== + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + +http-errors@~1.7.2: + version "1.7.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" + integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -1401,6 +1568,13 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" +iconv-lite@0.4.24: + version "0.4.24" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== + dependencies: + safer-buffer ">= 2.1.2 < 3" + ieee754@1.1.13, ieee754@^1.1.4: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" @@ -1431,7 +1605,7 @@ inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= -inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: +inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -1449,6 +1623,11 @@ into-stream@^5.1.1: from2 "^2.3.0" p-is-promise "^3.0.0" +ipaddr.js@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" + integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== + is-arrayish@^0.3.1: version "0.3.2" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" @@ -1764,11 +1943,26 @@ make-dir@^3.0.0: dependencies: semver "^6.0.0" +media-typer@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" + integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= + +merge-descriptors@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" + integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= + merge2@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== +methods@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= + micromatch@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.2.tgz#4fcb0999bf9fbc2fcbdd212f6d629b9a56c39259" @@ -1782,6 +1976,11 @@ mime-db@1.44.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== +mime-db@1.48.0: + version "1.48.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" + integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== + mime-types@^2.1.12, mime-types@~2.1.19: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" @@ -1789,6 +1988,18 @@ mime-types@^2.1.12, mime-types@~2.1.19: dependencies: mime-db "1.44.0" +mime-types@~2.1.24: + version "2.1.31" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" + integrity sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg== + dependencies: + mime-db "1.48.0" + +mime@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + mime@^2.2.0: version "2.4.6" resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" @@ -1833,6 +2044,11 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= +ms@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== + ms@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" @@ -1846,6 +2062,11 @@ multistream@^2.1.1: inherits "^2.0.1" readable-stream "^2.0.5" +negotiator@0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" + integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== + node-fetch@^2.3.0, node-fetch@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" @@ -1899,6 +2120,13 @@ object-hash@^2.0.1: resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea" integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg== +on-finished@~2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" + integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= + dependencies: + ee-first "1.1.1" + once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -1974,6 +2202,11 @@ package-json@^6.3.0: registry-url "^5.0.0" semver "^6.2.0" +parseurl@~1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" + integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== + path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" @@ -1984,6 +2217,11 @@ path-parse@^1.0.6: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== +path-to-regexp@0.1.7: + version "0.1.7" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" + integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w= + path-type@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" @@ -2079,6 +2317,14 @@ protobufjs@^6.8.6, protobufjs@^6.9.0: "@types/node" "^13.7.0" long "^4.0.0" +proxy-addr@~2.0.5: + version "2.0.7" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" + integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== + dependencies: + forwarded "0.2.0" + ipaddr.js "1.9.1" + psl@^1.1.28, psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" @@ -2114,6 +2360,11 @@ pupa@^2.0.1: dependencies: escape-goat "^2.0.0" +qs@6.7.0: + version "6.7.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" + integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== + qs@~6.5.2: version "6.5.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -2129,6 +2380,21 @@ querystringify@^2.1.1: resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== +range-parser@~1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + +raw-body@2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.0.tgz#a1ce6fb9c9bc356ca52e89256ab59059e13d0332" + integrity sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q== + dependencies: + bytes "3.1.0" + http-errors "1.7.2" + iconv-lite "0.4.24" + unpipe "1.0.0" + rc@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -2292,17 +2558,17 @@ run-parallel@^1.1.9: resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679" integrity sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q== +safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - -safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -2339,11 +2605,45 @@ semver@^7.1.3: resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== +send@0.17.1: + version "0.17.1" + resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" + integrity sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg== + dependencies: + debug "2.6.9" + depd "~1.1.2" + destroy "~1.0.4" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + fresh "0.5.2" + http-errors "~1.7.2" + mime "1.6.0" + ms "2.1.1" + on-finished "~2.3.0" + range-parser "~1.2.1" + statuses "~1.5.0" + +serve-static@1.14.1: + version "1.14.1" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" + integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.17.1" + set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= +setprototypeof@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" + integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== + signal-exit@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" @@ -2391,6 +2691,11 @@ stack-trace@0.0.x: resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" integrity sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA= +"statuses@>= 1.5.0 < 2", statuses@~1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" + integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= + stream-browserify@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-2.0.2.tgz#87521d38a44aa7ee91ce1cd2a47df0cb49dd660b" @@ -2513,6 +2818,11 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" +toidentifier@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" + integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== + touch@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.0.tgz#fe365f5f75ec9ed4e56825e0bb76d24ab74af83b" @@ -2581,6 +2891,14 @@ type-fest@^0.8.1: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== +type-is@~1.6.17, type-is@~1.6.18: + version "1.6.18" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" + integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== + dependencies: + media-typer "0.3.0" + mime-types "~2.1.24" + typedarray-to-buffer@^3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" @@ -2636,6 +2954,11 @@ universalify@^1.0.0: resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d" integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug== +unpipe@1.0.0, unpipe@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" + integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= + update-notifier@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.1.tgz#895fc8562bbe666179500f9f2cebac4f26323746" @@ -2705,6 +3028,11 @@ util@^0.11.1: dependencies: inherits "2.0.3" +utils-merge@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" + integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= + uuid-parse@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/uuid-parse/-/uuid-parse-1.1.0.tgz#7061c5a1384ae0e1f943c538094597e1b5f3a65b" @@ -2735,6 +3063,11 @@ validator@^9.4.1: resolved "https://registry.yarnpkg.com/validator/-/validator-9.4.1.tgz#abf466d398b561cd243050112c6ff1de6cc12663" integrity sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA== +vary@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" + integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= + verror@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" From d117cec09e25c52ea69807a4d4db051505a3b934 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 07:42:21 +0300 Subject: [PATCH 038/101] js-executor: partitions_consumed_concurrently marked as experimental --- msa/js-executor/config/custom-environment-variables.yml | 2 +- msa/js-executor/config/default.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 4185e65835..fdf261e25c 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -29,7 +29,7 @@ kafka: acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "TB_KAFKA_BATCH_SIZE" # for producer linger_ms: "TB_KAFKA_LINGER_MS" # for producer - partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency + partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 190cf097ed..7a6e7c2469 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -29,7 +29,7 @@ kafka: acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "128" # for producer linger_ms: "1" # for producer - partitions_consumed_concurrently: "1" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency + partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "30000" # The default value in kafkajs is: 30000 compression: "gzip" # gzip or uncompressed topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1" From a3b31337cae7f0b639285855fd1cc050d05cfda4 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 09:09:12 +0300 Subject: [PATCH 039/101] sync method replaced with async executeGenerateAsync for ScriptEngine api. affected Generator node, ruleChainController (merged with ce) --- .../controller/RuleChainController.java | 4 +- .../service/script/RemoteJsInvokeService.java | 8 ++- .../script/RuleNodeJsScriptEngine.java | 19 ++++--- .../rule/engine/api/ScriptEngine.java | 2 +- .../rule/engine/debug/TbMsgGeneratorNode.java | 50 ++++++++++++------- 5 files changed, 54 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index d5fcc2706f..3b9af2f0f2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -74,6 +74,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -86,6 +87,7 @@ public class RuleChainController extends BaseController { public static final String RULE_NODE_ID = "ruleNodeId"; private static final ObjectMapper objectMapper = new ObjectMapper(); + public static final int TIMEOUT = 20; @Autowired private InstallScripts installScripts; @@ -391,7 +393,7 @@ public class RuleChainController extends BaseController { output = msgToOutput(engine.executeUpdate(inMsg)); break; case "generate": - output = msgToOutput(engine.executeGenerate(inMsg)); + output = msgToOutput(engine.executeGenerateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS)); break; case "filter": boolean result = engine.executeFilter(inMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 334a471973..1bc9203333 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -26,6 +25,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -161,6 +161,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override protected ListenableFuture doInvokeFunction(UUID scriptId, String functionName, Object[] args) { + log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptId, maxRequestsTimeout); String scriptBody = scriptIdToBodysMap.get(scriptId); if (scriptBody == null) { return Futures.immediateFailedFuture(new RuntimeException("No script body found for scriptId: [" + scriptId + "]!")); @@ -180,6 +181,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); if (maxRequestsTimeout > 0) { future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService); @@ -201,6 +205,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, callbackExecutor); return Futures.transform(future, response -> { + stopWatch.stop(); + log.trace("doInvokeFunction js-response took {}ms for uuid {}", stopWatch.getTotalTimeMillis(), response.getKey()); JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 066ce71a58..c688d6978d 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -102,7 +102,6 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); return TbMsg.transformMsg(msg, newMessageType, msg.getOriginator(), newMetadata, newData); } catch (Throwable th) { - th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); } } @@ -141,13 +140,16 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } @Override - public TbMsg executeGenerate(TbMsg prevMsg) throws ScriptException { - JsonNode result = executeScript(prevMsg); - if (!result.isObject()) { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); - } - return unbindMsg(result, prevMsg); + public ListenableFuture executeGenerateAsync(TbMsg prevMsg) { + log.trace("execute generate async, prevMsg {}", prevMsg); + return Futures.transformAsync(executeScriptAsync(prevMsg), result -> { + if (!result.isObject()) { + log.warn("Wrong result type: {}", result.getNodeType()); + throw new ScriptException("Wrong result type: " + result.getNodeType()); + } + return Futures.immediateFuture(unbindMsg(result, prevMsg)); + }, MoreExecutors.directExecutor()); + } @Override @@ -234,6 +236,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } private ListenableFuture executeScriptAsync(TbMsg msg) { + log.trace("execute script async, msg {}", msg); String[] inArgs = prepareArgs(msg); return Futures.transformAsync(sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]), o -> { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index 49420e8feb..a5dcf535e4 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -29,7 +29,7 @@ public interface ScriptEngine { ListenableFuture> executeUpdateAsync(TbMsg msg); - TbMsg executeGenerate(TbMsg prevMsg) throws ScriptException; + ListenableFuture executeGenerateAsync(TbMsg prevMsg); boolean executeFilter(TbMsg msg) throws ScriptException; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index b5a47b4a0d..8fa4b97063 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -15,9 +15,12 @@ */ package org.thingsboard.rule.engine.debug; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.util.StringUtils; +import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.TbContext; @@ -35,6 +38,7 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -64,10 +68,11 @@ public class TbMsgGeneratorNode implements TbNode { private EntityId originatorId; private UUID nextTickId; private TbMsg prevMsg; - private volatile boolean initialized; + private final AtomicBoolean initialized = new AtomicBoolean(false); @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + log.trace("init generator with config {}", configuration); this.config = TbNodeUtils.convert(configuration, TbMsgGeneratorNodeConfiguration.class); this.delay = TimeUnit.SECONDS.toMillis(config.getPeriodInSeconds()); this.currentMsgCount = 0; @@ -81,35 +86,39 @@ public class TbMsgGeneratorNode implements TbNode { @Override public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { + log.trace("onPartitionChangeMsg, PartitionChangeMsg {}, config {}", msg, config); updateGeneratorState(ctx); } private void updateGeneratorState(TbContext ctx) { + log.trace("updateGeneratorState, config {}", config); if (ctx.isLocalEntity(originatorId)) { - if (!initialized) { - initialized = true; + if (initialized.compareAndSet(false, true)) { this.jsEngine = ctx.createJsScriptEngine(config.getJsScript(), "prevMsg", "prevMetadata", "prevMsgType"); scheduleTickMsg(ctx); } - } else if (initialized) { - initialized = false; + } else if (initialized.compareAndSet(true, false)) { destroy(); } } @Override public void onMsg(TbContext ctx, TbMsg msg) { - if (initialized && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) { + log.trace("onMsg, config {}, msg {}", config, msg); + if (initialized.get() && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) { + TbStopWatch sw = TbStopWatch.startNew(); withCallback(generate(ctx, msg), m -> { - if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { + log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); + if (initialized.get() && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { ctx.enqueueForTellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } }, t -> { - if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { + log.warn("onMsg onFailure callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); + if (initialized.get() && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { ctx.tellFailure(msg, t); scheduleTickMsg(ctx); currentMsgCount++; @@ -119,6 +128,7 @@ public class TbMsgGeneratorNode implements TbNode { } private void scheduleTickMsg(TbContext ctx) { + log.trace("scheduleTickMsg, config {}", config); long curTs = System.currentTimeMillis(); if (lastScheduledTs == 0L) { lastScheduledTs = curTs; @@ -131,22 +141,26 @@ public class TbMsgGeneratorNode implements TbNode { } private ListenableFuture generate(TbContext ctx, TbMsg msg) { - return ctx.getJsExecutor().executeAsync(() -> { - if (prevMsg == null) { - prevMsg = ctx.newMsg(ServiceQueue.MAIN, "", originatorId, msg.getCustomerId(), new TbMsgMetaData(), "{}"); - } - if (initialized) { - ctx.logJsEvalRequest(); - TbMsg generated = jsEngine.executeGenerate(prevMsg); + log.trace("generate, config {}", config); + if (prevMsg == null) { + prevMsg = ctx.newMsg(ServiceQueue.MAIN, "", originatorId, msg.getCustomerId(), new TbMsgMetaData(), "{}"); + } + if (initialized.get()) { + ctx.logJsEvalRequest(); + return Futures.transformAsync(jsEngine.executeGenerateAsync(prevMsg), generated -> { + log.trace("generate process response, generated {}, config {}", generated, config); ctx.logJsEvalResponse(); prevMsg = ctx.newMsg(ServiceQueue.MAIN, generated.getType(), originatorId, msg.getCustomerId(), generated.getMetaData(), generated.getData()); - } - return prevMsg; - }); + return Futures.immediateFuture(prevMsg); + }, MoreExecutors.directExecutor()); + } + return Futures.immediateFuture(prevMsg); + } @Override public void destroy() { + log.trace("destroy, config {}", config); prevMsg = null; if (jsEngine != null) { jsEngine.destroy(); From 6888a1b349de234db1f6f3a8b81326fb34f438b1 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 08:22:38 +0300 Subject: [PATCH 040/101] new utility class tbStopWatch added to improve performance measurement experience --- common/util/pom.xml | 5 ++ .../thingsboard/common/util/TbStopWatch.java | 62 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java diff --git a/common/util/pom.xml b/common/util/pom.xml index 6f66da5790..128bc478d5 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -36,6 +36,11 @@ + + org.springframework + spring-core + ${spring.version} + com.google.guava guava diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java new file mode 100644 index 0000000000..b7dbd0052c --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -0,0 +1,62 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.common.util; + +import org.springframework.util.StopWatch; + +/** + * Utility method that extends Spring Framework StopWatch + * It is a MONOTONIC time stopwatch. + * It is a replacement for any measurements with a wall-clock like System.currentTimeMillis() + * It is not affected by leap second, day-light saving and wall-clock adjustments by manual or network time synchronization + * The main features is a single call for common use cases: + * - create and start: TbStopWatch sw = TbStopWatch.startNew() + * - stop and get: sw.stopAndGetTotalTimeMillis() or sw.stopAndGetLastTaskTimeMillis() + * */ +public class TbStopWatch extends StopWatch { + + public static TbStopWatch startNew(){ + TbStopWatch stopWatch = new TbStopWatch(); + stopWatch.start(); + return stopWatch; + } + + public long stopAndGetTotalTimeMillis(){ + stop(); + return getTotalTimeMillis(); + } + + public long stopAndGetLastTaskTimeMillis(){ + stop(); + return getLastTaskTimeMillis(); + } + +} From f4c98595cb973f1c74f76c647d7bf4c4334851e9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 11:07:59 +0300 Subject: [PATCH 041/101] generator node: comment added on direct executor --- .../org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 8fa4b97063..ab9c68e41f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -152,7 +152,7 @@ public class TbMsgGeneratorNode implements TbNode { ctx.logJsEvalResponse(); prevMsg = ctx.newMsg(ServiceQueue.MAIN, generated.getType(), originatorId, msg.getCustomerId(), generated.getMetaData(), generated.getData()); return Futures.immediateFuture(prevMsg); - }, MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); //usually it runs on js-executor-remote-callback thread pool } return Futures.immediateFuture(prevMsg); From 585f473bdaa9477fd25616982dcc34ba80d50ed9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 15:00:00 +0300 Subject: [PATCH 042/101] exception message added onFailure for GeneratorNode --- .../org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index ab9c68e41f..cb9e25d34b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -117,7 +117,7 @@ public class TbMsgGeneratorNode implements TbNode { } }, t -> { - log.warn("onMsg onFailure callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); + log.warn("onMsg onFailure callback, took {}ms, config {}, msg {}, exception {}", sw.stopAndGetTotalTimeMillis(), config, msg, t); if (initialized.get() && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { ctx.tellFailure(msg, t); scheduleTickMsg(ctx); From a04eac6015bc98e626f446cfa9b6c4be802df626 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 27 Apr 2021 18:26:52 +0300 Subject: [PATCH 043/101] added logs and timing metrics for DefaultTbQueueRequestTemplate, TbKafkaConsumerTemplate (todo revert or refactor) --- .../queue/common/DefaultTbQueueRequestTemplate.java | 8 ++++---- .../server/queue/kafka/TbKafkaConsumerTemplate.java | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index b171f2d8d8..e07be52491 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -92,10 +92,10 @@ public class DefaultTbQueueRequestTemplate responses = responseTemplate.poll(pollInterval); - if (responses.size() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.size()); - } + final int pendingRequestsCount = pendingRequests.size(); + log.trace("Starting template pool topic {}, for pendingRequests {}", responseTemplate.getTopic(), pendingRequestsCount); + List responses = responseTemplate.poll(pollInterval); //poll js responses + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); responses.forEach(response -> { byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); UUID requestId; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index fe4300a421..816a732d41 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.springframework.util.StopWatch; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; @@ -82,7 +83,16 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue @Override protected List> doPoll(long durationInMillis) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + + log.trace("poll topic {} maxDuration {}", getTopic(), durationInMillis); + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + + stopWatch.stop(); + log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + if (records.isEmpty()) { return Collections.emptyList(); } else { From 67c9025a0625270f02626421d5a47d6c6c79ee3c Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 18:01:52 +0300 Subject: [PATCH 044/101] improved logs for timeouts for DefaultTbQueueRequestTemplate --- .../server/queue/common/DefaultTbQueueRequestTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e07be52491..084b43f4b6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -121,7 +121,7 @@ public class DefaultTbQueueRequestTemplate staleRequest = pendingRequests.remove(key); if (staleRequest != null) { - log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); staleRequest.future.setException(new TimeoutException()); } } @@ -129,7 +129,7 @@ public class DefaultTbQueueRequestTemplate Date: Wed, 5 May 2021 20:10:51 +0300 Subject: [PATCH 045/101] refactored for tests DefaultTbQueueRequestTemplate class --- .../common/DefaultTbQueueRequestTemplate.java | 112 ++++++++++-------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 084b43f4b6..2c779d5d5a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -85,59 +85,75 @@ public class DefaultTbQueueRequestTemplate { - long nextCleanupMs = 0L; - while (!stopped) { - try { - final int pendingRequestsCount = pendingRequests.size(); - log.trace("Starting template pool topic {}, for pendingRequests {}", responseTemplate.getTopic(), pendingRequestsCount); - List responses = responseTemplate.poll(pollInterval); //poll js responses - log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); - responses.forEach(response -> { - byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); - UUID requestId; - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header and body", response); - } else { - requestId = bytesToUuid(requestIdHeader); - log.trace("[{}] Response received: {}", requestId, response); - ResponseMetaData expectedResponse = pendingRequests.remove(requestId); - if (expectedResponse == null) { - log.trace("[{}] Invalid or stale request", requestId); - } else { - expectedResponse.future.set(response); + executor.submit(this::fetchAndProcessResponses); + } + + void fetchAndProcessResponses() { + long nextCleanupMs = 0L; + while (!stopped) { + try { + final int pendingRequestsCount = pendingRequests.size(); + log.trace("Starting template pool topic {}, for pendingRequests {}", responseTemplate.getTopic(), pendingRequestsCount); + List responses = doPoll(); //poll js responses + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); + responses.forEach(this::processResponse); + responseTemplate.commit(); + tickTs = System.currentTimeMillis(); + tickSize = pendingRequests.size(); + if (nextCleanupMs < tickTs) { + //cleanup; + pendingRequests.forEach((key, value) -> { + if (value.expTime < tickTs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + setTimeoutException(key, staleRequest); } } }); - responseTemplate.commit(); - tickTs = System.currentTimeMillis(); - tickSize = pendingRequests.size(); - if (nextCleanupMs < tickTs) { - //cleanup; - pendingRequests.forEach((key, value) -> { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); - staleRequest.future.setException(new TimeoutException()); - } - } - }); - nextCleanupMs = tickTs + maxRequestTimeout; - } - } catch (Throwable e) { - log.warn("Failed to obtain responses from queue. Going to sleep " + pollInterval + "ms", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new responses", e2); - } + nextCleanupMs = tickTs + maxRequestTimeout; } + } catch (Throwable e) { + log.warn("Failed to obtain responses from queue. Going to sleep " + pollInterval + "ms", e); + sleep(); } - }); + } + } + + List doPoll() { + return responseTemplate.poll(pollInterval); + } + + void sleep() { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new responses", e2); + } + } + + void setTimeoutException(UUID key, ResponseMetaData staleRequest) { + log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + staleRequest.future.setException(new TimeoutException()); + } + + void processResponse(Response response) { + byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); + UUID requestId; + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header and body", response); + } else { + requestId = bytesToUuid(requestIdHeader); + log.trace("[{}] Response received: {}", requestId, response); + ResponseMetaData expectedResponse = pendingRequests.remove(requestId); + if (expectedResponse == null) { + log.warn("[{}] Invalid or stale request, response: {}", requestId, response); + } else { + expectedResponse.future.set(response); + } + } } @Override @@ -174,7 +190,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); + log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } @@ -184,7 +200,7 @@ public class DefaultTbQueueRequestTemplate Date: Thu, 6 May 2021 11:48:42 +0300 Subject: [PATCH 046/101] test: added init stop test for DefaultTbQueueRequestTemplate --- .../common/DefaultTbQueueRequestTemplate.java | 22 ++-- .../DefaultTbQueueRequestTemplateTest.java | 113 ++++++++++++++++++ 2 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 2c779d5d5a..a51e143e26 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -31,6 +31,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.common.stats.MessagesStats; +import javax.annotation.Nullable; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -47,14 +48,14 @@ public class DefaultTbQueueRequestTemplate requestTemplate; private final TbQueueConsumer responseTemplate; private final ConcurrentMap> pendingRequests; - private final boolean internalExecutor; + final boolean internalExecutor; private final ExecutorService executor; private final long maxRequestTimeout; private final long maxPendingRequests; private final long pollInterval; - private volatile long tickTs = 0L; - private volatile long tickSize = 0L; - private volatile boolean stopped = false; + volatile long tickTs = 0L; + volatile long tickSize = 0L; + volatile boolean stopped = false; private MessagesStats messagesStats; @@ -65,7 +66,7 @@ public class DefaultTbQueueRequestTemplate requestTemplate; + @Mock + TbQueueConsumer responseTemplate; + + long maxRequestTimeout = 20; + long maxPendingRequests = 10000; + long pollInterval = 25; + String topic = "js-responses-tb-node-0"; + + DefaultTbQueueRequestTemplate inst; + + @Before + public void setUp() throws Exception { + willReturn(topic).given(responseTemplate).getTopic(); + inst = new DefaultTbQueueRequestTemplate( + queueAdmin, requestTemplate, responseTemplate, + maxRequestTimeout, maxPendingRequests, pollInterval, executor); + + } + + @Test + public void givenExternalExecutor_whenStop_thenDoNotShutdownExecutor() { + inst = Mockito.spy(inst); + willDoNothing().given(inst).fetchAndProcessResponses(); + Assert.assertFalse(inst.stopped); + + inst.init(); + Assert.assertNotEquals(inst.tickTs, 0); + Assert.assertFalse(inst.internalExecutor); + verify(queueAdmin, times(1)).createTopicIfNotExists(topic); + verify(requestTemplate, times(1)).init(); + verify(responseTemplate, times(1)).subscribe(); + verify(executor, times(1)).submit(any(Runnable.class)); + + inst.stop(); + Assert.assertTrue(inst.stopped); + verify(responseTemplate, times(1)).unsubscribe(); + verify(requestTemplate, times(1)).stop(); + verify(executor, never()).shutdownNow(); + } + + @Test + public void fetchAndProcessResponses() { + inst.init(); + + inst.stop(); + } +} \ No newline at end of file From 872717828cc0e9338f6d1e986abcb67a56e0d06f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 6 May 2021 14:54:27 +0300 Subject: [PATCH 047/101] test: refactored and added mainLoop test DefaultTbQueueRequestTemplate --- .../DefaultTbQueueRequestTemplateTest.java | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 769b66fcd4..413b90c5b9 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -31,27 +31,27 @@ package org.thingsboard.server.queue.common; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.BDDMockito; import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.internal.util.reflection.Whitebox; import org.mockito.runners.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; +import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -61,16 +61,17 @@ public class DefaultTbQueueRequestTemplateTest { @Mock TbQueueAdmin queueAdmin; @Mock - ExecutorService executor; - @Mock TbQueueProducer requestTemplate; @Mock TbQueueConsumer responseTemplate; + @Mock + ExecutorService executorMock; + ExecutorService executor; + String topic = "js-responses-tb-node-0"; long maxRequestTimeout = 20; long maxPendingRequests = 10000; long pollInterval = 25; - String topic = "js-responses-tb-node-0"; DefaultTbQueueRequestTemplate inst; @@ -79,35 +80,70 @@ public class DefaultTbQueueRequestTemplateTest { willReturn(topic).given(responseTemplate).getTopic(); inst = new DefaultTbQueueRequestTemplate( queueAdmin, requestTemplate, responseTemplate, - maxRequestTimeout, maxPendingRequests, pollInterval, executor); + maxRequestTimeout, maxPendingRequests, pollInterval, executorMock); + + } + @After + public void tearDown() throws Exception { + if (executor != null) { + executor.shutdownNow(); + } } @Test - public void givenExternalExecutor_whenStop_thenDoNotShutdownExecutor() { - inst = Mockito.spy(inst); - willDoNothing().given(inst).fetchAndProcessResponses(); - Assert.assertFalse(inst.stopped); + public void givenInstance_whenVerifyInitialParameters_thenOK() { + assertEquals(maxPendingRequests, inst.maxPendingRequests); + assertEquals(maxRequestTimeout, inst.maxRequestTimeout); + assertEquals(pollInterval, inst.pollInterval); + assertEquals(executorMock, inst.executor); + assertFalse(inst.stopped); + assertFalse(inst.internalExecutor); + } + + @Test + public void givenExternalExecutor_whenInitStop_thenOK() { + inst = spy(inst); + willDoNothing().given(inst).mainLoop(); inst.init(); - Assert.assertNotEquals(inst.tickTs, 0); - Assert.assertFalse(inst.internalExecutor); + assertNotEquals(0, inst.tickTs); + assertEquals(0, inst.nextCleanupMs); verify(queueAdmin, times(1)).createTopicIfNotExists(topic); verify(requestTemplate, times(1)).init(); verify(responseTemplate, times(1)).subscribe(); - verify(executor, times(1)).submit(any(Runnable.class)); + verify(executorMock, times(1)).submit(any(Runnable.class)); inst.stop(); - Assert.assertTrue(inst.stopped); + assertTrue(inst.stopped); verify(responseTemplate, times(1)).unsubscribe(); verify(requestTemplate, times(1)).stop(); - verify(executor, never()).shutdownNow(); + verify(executorMock, never()).shutdownNow(); } @Test - public void fetchAndProcessResponses() { - inst.init(); + public void givenMainLoop_whenLoopFewTimes_thenVerifyInvocationCount() throws InterruptedException { + inst = spy(inst); + executor = inst.createExecutor(); + CountDownLatch latch = new CountDownLatch(5); + willDoNothing().given(inst).sleep(); + willAnswer(invocation -> { + if (latch.getCount() == 1) { + inst.stop(); //stop the loop in natural way + } + if (latch.getCount() == 3 || latch.getCount() == 4) { + latch.countDown(); + throw new RuntimeException("test catch block"); + } + latch.countDown(); + return null; + }).given(inst).fetchAndProcessResponses(); - inst.stop(); + executor.submit(inst::mainLoop); + latch.await(10, TimeUnit.SECONDS); + + verify(inst, times(5)).fetchAndProcessResponses(); + verify(inst, times(2)).sleep(); } + } \ No newline at end of file From d5fffa5002528db63302749b354aff8cd488ce5d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 6 May 2021 15:50:07 +0300 Subject: [PATCH 048/101] test: added failed test case with overflow maxPendingRequests for DefaultTbQueueRequestTemplate.send(). another refactoring for easy mocking. --- .../DefaultTbQueueRequestTemplateTest.java | 53 ++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 413b90c5b9..ef9d0835d2 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -39,17 +39,22 @@ import org.mockito.runners.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.TbQueueProducer; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -70,7 +75,7 @@ public class DefaultTbQueueRequestTemplateTest { ExecutorService executor; String topic = "js-responses-tb-node-0"; long maxRequestTimeout = 20; - long maxPendingRequests = 10000; + long maxPendingRequests = 32; long pollInterval = 25; DefaultTbQueueRequestTemplate inst; @@ -78,9 +83,9 @@ public class DefaultTbQueueRequestTemplateTest { @Before public void setUp() throws Exception { willReturn(topic).given(responseTemplate).getTopic(); - inst = new DefaultTbQueueRequestTemplate( + inst = spy(new DefaultTbQueueRequestTemplate( queueAdmin, requestTemplate, responseTemplate, - maxRequestTimeout, maxPendingRequests, pollInterval, executorMock); + maxRequestTimeout, maxPendingRequests, pollInterval, executorMock)); } @@ -103,7 +108,6 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenExternalExecutor_whenInitStop_thenOK() { - inst = spy(inst); willDoNothing().given(inst).mainLoop(); inst.init(); @@ -123,7 +127,6 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenMainLoop_whenLoopFewTimes_thenVerifyInvocationCount() throws InterruptedException { - inst = spy(inst); executor = inst.createExecutor(); CountDownLatch latch = new CountDownLatch(5); willDoNothing().given(inst).sleep(); @@ -146,4 +149,42 @@ public class DefaultTbQueueRequestTemplateTest { verify(inst, times(2)).sleep(); } + @Test + public void givenMessages_whenSend_thenOK() { + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + inst.init(); + int msgCount = 10; + for (int i = 0; i < msgCount; i++) { + inst.send(getRequestMsgMock()); + } + assertEquals(msgCount, inst.pendingRequests.size()); + verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); + } + + @Test + public void givenMessagesOverMaxPendingRequests_whenSend_thenImmediateFailedFutureForTheOfRequests() { + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + inst.init(); + assertEquals(0, inst.tickSize); + int msgOverflowCount = 10; + for (int i = 0; i < inst.maxPendingRequests; i++) { + assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + } + for (int i = 0; i < msgOverflowCount; i++) { + assertTrue("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future + } + assertEquals(inst.maxPendingRequests, inst.pendingRequests.size()); + verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); + } + + @Test + public void givenNothing_whenFetchAndProcessResponsesWithTimeout_thenFail() { + + } + + TbQueueMsg getRequestMsgMock() { + TbQueueMsg requestMsg = mock(TbQueueMsg.class); + willReturn(mock(TbQueueMsgHeaders.class)).given(requestMsg).getHeaders(); + return requestMsg; + } } \ No newline at end of file From 9666986156a0005f2d934646d625b29c02aa091f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 10:04:41 +0300 Subject: [PATCH 049/101] test: added failed test for FetchAndProcessResponses when request removed as staled too early DefaultTbQueueRequestTemplate --- .../common/DefaultTbQueueRequestTemplate.java | 131 ++++++++++++------ .../DefaultTbQueueRequestTemplateTest.java | 60 ++++++-- 2 files changed, 138 insertions(+), 53 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index a51e143e26..bfda553538 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -47,15 +48,16 @@ public class DefaultTbQueueRequestTemplate requestTemplate; private final TbQueueConsumer responseTemplate; - private final ConcurrentMap> pendingRequests; + final ConcurrentMap> pendingRequests; final boolean internalExecutor; - private final ExecutorService executor; - private final long maxRequestTimeout; - private final long maxPendingRequests; - private final long pollInterval; + final ExecutorService executor; + final long maxRequestTimeout; + final long maxPendingRequests; + final long pollInterval; volatile long tickTs = 0L; volatile long tickSize = 0L; volatile boolean stopped = false; + long nextCleanupMs = 0L; private MessagesStats messagesStats; @@ -75,44 +77,26 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses - log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); - responses.forEach(this::processResponse); - responseTemplate.commit(); - tickTs = System.currentTimeMillis(); - tickSize = pendingRequests.size(); - if (nextCleanupMs < tickTs) { - //cleanup; - pendingRequests.forEach((key, value) -> { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - setTimeoutException(key, staleRequest); - } - } - }); - nextCleanupMs = tickTs + maxRequestTimeout; - } + fetchAndProcessResponses(); } catch (Throwable e) { log.warn("Failed to obtain responses from queue. Going to sleep " + pollInterval + "ms", e); sleep(); @@ -120,6 +104,36 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses + //if (responses.size() > 0) { + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); + //} + responses.forEach(this::processResponse); //this can take a long time + responseTemplate.commit(); + tickTs = getCurrentTime(); + tickSize = pendingRequests.size(); + if (nextCleanupMs < tickTs) { + //cleanup; + pendingRequests.forEach((key, value) -> { + if (value.expTime < tickTs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + setTimeoutException(key, staleRequest, tickTs); + } + } + }); + setupNextCleanup(); + } + } + + void setupNextCleanup() { + nextCleanupMs = tickTs + maxRequestTimeout; + log.info("setupNextCleanup {}", nextCleanupMs); + } + List doPoll() { return responseTemplate.poll(pollInterval); } @@ -132,8 +146,13 @@ public class DefaultTbQueueRequestTemplate staleRequest) { - log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + void setTimeoutException(UUID key, ResponseMetaData staleRequest, long tickTs) { + if (tickTs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { + log.info("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + } else { + log.error("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + } + staleRequest.future.setException(new TimeoutException()); } @@ -144,10 +163,10 @@ public class DefaultTbQueueRequestTemplate expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { - log.warn("[{}] Invalid or stale request, response: {}", requestId, response); + log.warn("[{}] Invalid or stale request, response: {}", requestId, String.valueOf(response).replace("\n", " ")); } else { expectedResponse.future.set(response); } @@ -184,11 +203,22 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); + ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future, currentTime, maxRequestTimeout); + log.info("pending {}", responseMetaData); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); + sendToRequestTemplate(request, requestId, future, responseMetaData); + return future; + } + + long getCurrentTime() { + return System.currentTimeMillis(); + } + + void sendToRequestTemplate(Request request, UUID requestId, SettableFuture future, ResponseMetaData responseMetaData) { + log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, String.valueOf(request).replace("\n", " ")); if (messagesStats != null) { messagesStats.incrementTotal(); } @@ -198,7 +228,7 @@ public class DefaultTbQueueRequestTemplate { + @Getter + static class ResponseMetaData { + private final long submitTime; + private final long timeout; private final long expTime; private final SettableFuture future; - ResponseMetaData(long ts, SettableFuture future) { + ResponseMetaData(long ts, SettableFuture future, long submitTime, long timeout) { + this.submitTime = submitTime; + this.timeout = timeout; this.expTime = ts; this.future = future; } + + @Override + public String toString() { + return "ResponseMetaData{" + + "submitTime=" + submitTime + + ", calculatedExpTime=" + (submitTime + timeout) + + ", expTime=" + expTime + + ", deltaMs=" + (expTime - submitTime) + + ", future=" + future + + '}'; + } } } diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index ef9d0835d2..715b4eab21 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -29,23 +29,29 @@ * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. */ package org.thingsboard.server.queue.common; - +import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.TbQueueProducer; +import java.util.Collections; +import java.util.List; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -54,12 +60,17 @@ import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.hamcrest.MatcherAssert.assertThat; + +@Slf4j @RunWith(MockitoJUnitRunner.class) public class DefaultTbQueueRequestTemplateTest { @@ -74,9 +85,9 @@ public class DefaultTbQueueRequestTemplateTest { ExecutorService executor; String topic = "js-responses-tb-node-0"; - long maxRequestTimeout = 20; + long maxRequestTimeout = 10; long maxPendingRequests = 32; - long pollInterval = 25; + long pollInterval = 5; DefaultTbQueueRequestTemplate inst; @@ -171,20 +182,49 @@ public class DefaultTbQueueRequestTemplateTest { assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only } for (int i = 0; i < msgOverflowCount; i++) { - assertTrue("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future + assertFalse("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future } - assertEquals(inst.maxPendingRequests, inst.pendingRequests.size()); + assertThat(inst.pendingRequests.size(), equalTo(inst.maxPendingRequests)); verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); } @Test - public void givenNothing_whenFetchAndProcessResponsesWithTimeout_thenFail() { + public void givenNothing_whenSendAndFetchAndProcessResponsesWithTimeout_thenFail() { + //given + AtomicLong currentTime = new AtomicLong(); + willAnswer(x -> { + log.info("currentTime={}", currentTime.get()); + return currentTime.get(); + }).given(inst).getCurrentTime(); + inst.init(); + inst.setupNextCleanup(); + willReturn(Collections.emptyList()).given(inst).doPoll(); + willDoNothing().given(inst).processResponse(any()); + + //when + for (int i = 0; i <= inst.maxRequestTimeout*2; i++) { + currentTime.incrementAndGet(); + assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + if (i % (inst.maxRequestTimeout * 3 / 2) == 0) { + inst.fetchAndProcessResponses(); + } + } + + //then + ArgumentCaptor argumentCaptorResp = ArgumentCaptor.forClass(DefaultTbQueueRequestTemplate.ResponseMetaData.class); + ArgumentCaptor argumentCaptorUUID = ArgumentCaptor.forClass(UUID.class); + ArgumentCaptor argumentCaptorLong = ArgumentCaptor.forClass(Long.class); + verify(inst, atLeastOnce()).setTimeoutException(argumentCaptorUUID.capture(), argumentCaptorResp.capture(), argumentCaptorLong.capture()); + + List responseMetaDataList = argumentCaptorResp.getAllValues(); + List tickTsList = argumentCaptorLong.getAllValues(); + for (int i = 0; i < responseMetaDataList.size(); i++) { + assertThat("tickTs >= calculatedExpTime", tickTsList.get(i), greaterThanOrEqualTo(responseMetaDataList.get(i).getSubmitTime() + responseMetaDataList.get(i).getTimeout())); + } } TbQueueMsg getRequestMsgMock() { - TbQueueMsg requestMsg = mock(TbQueueMsg.class); - willReturn(mock(TbQueueMsgHeaders.class)).given(requestMsg).getHeaders(); - return requestMsg; + return mock(TbQueueMsg.class, RETURNS_DEEP_STUBS); } } \ No newline at end of file From 28235732c6140dfc9a5cb81d4c2e39b8ac217532 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 12:07:17 +0300 Subject: [PATCH 050/101] test: fixed class DefaultTbQueueRequestTemplate --- .../common/DefaultTbQueueRequestTemplate.java | 90 +++++++++++-------- .../DefaultTbQueueRequestTemplateTest.java | 37 ++++---- 2 files changed, 73 insertions(+), 54 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index bfda553538..d5179385e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -36,10 +36,12 @@ import javax.annotation.Nullable; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; @Slf4j public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate @@ -48,16 +50,15 @@ public class DefaultTbQueueRequestTemplate requestTemplate; private final TbQueueConsumer responseTemplate; - final ConcurrentMap> pendingRequests; + final ConcurrentHashMap> pendingRequests = new ConcurrentHashMap<>(); final boolean internalExecutor; final ExecutorService executor; - final long maxRequestTimeout; + final long maxRequestTimeoutNs; final long maxPendingRequests; final long pollInterval; - volatile long tickTs = 0L; - volatile long tickSize = 0L; volatile boolean stopped = false; - long nextCleanupMs = 0L; + long nextCleanupNs = 0L; + private final Lock cleanerLock = new ReentrantLock(); private MessagesStats messagesStats; @@ -72,8 +73,7 @@ public class DefaultTbQueueRequestTemplate(); - this.maxRequestTimeout = maxRequestTimeout; + this.maxRequestTimeoutNs = TimeUnit.MILLISECONDS.toNanos(maxRequestTimeout); this.maxPendingRequests = maxPendingRequests; this.pollInterval = pollInterval; this.internalExecutor = (executor == null); @@ -88,7 +88,6 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses //if (responses.size() > 0) { @@ -113,25 +112,36 @@ public class DefaultTbQueueRequestTemplate { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - setTimeoutException(key, staleRequest, tickTs); + tryCleanStaleRequests(); + } + + private boolean tryCleanStaleRequests() { + if (!cleanerLock.tryLock()) { + return false; + } + try { + log.trace("tryCleanStaleRequest..."); + final long currentNs = getCurrentClockNs(); + if (nextCleanupNs < currentNs) { + pendingRequests.forEach((key, value) -> { + if (value.expTime < currentNs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + setTimeoutException(key, staleRequest, currentNs); + } } - } - }); - setupNextCleanup(); + }); + setupNextCleanup(); + } + } finally { + cleanerLock.unlock(); } + return true; } void setupNextCleanup() { - nextCleanupMs = tickTs + maxRequestTimeout; - log.info("setupNextCleanup {}", nextCleanupMs); + nextCleanupNs = getCurrentClockNs() + maxRequestTimeoutNs; + log.info("setupNextCleanup {}", nextCleanupNs); } List doPoll() { @@ -146,11 +156,11 @@ public class DefaultTbQueueRequestTemplate staleRequest, long tickTs) { - if (tickTs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { - log.info("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + void setTimeoutException(UUID key, ResponseMetaData staleRequest, long currentNs) { + if (currentNs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { + log.info("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } else { - log.error("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + log.error("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } staleRequest.future.setException(new TimeoutException()); @@ -197,23 +207,31 @@ public class DefaultTbQueueRequestTemplate send(Request request) { - if (tickSize > maxPendingRequests) { + if (pendingRequests.mappingCount() >= maxPendingRequests) { + log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance", maxPendingRequests); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } UUID requestId = UUID.randomUUID(); request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); - long currentTime = getCurrentTime(); - request.getHeaders().put(REQUEST_TIME, longToBytes(currentTime)); + request.getHeaders().put(REQUEST_TIME, longToBytes(getCurrentTimeMs())); + long currentClockNs = getCurrentClockNs(); SettableFuture future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future, currentTime, maxRequestTimeout); - log.info("pending {}", responseMetaData); - pendingRequests.putIfAbsent(requestId, responseMetaData); + ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + maxRequestTimeoutNs, future, currentClockNs, maxRequestTimeoutNs); + log.info("pending {}", responseMetaData); //TODO trace + if (pendingRequests.putIfAbsent(requestId, responseMetaData) != null) { + log.warn("Pending request already exists [{}]!", maxPendingRequests); + return Futures.immediateFailedFuture(new RuntimeException("Pending request already exists !" + requestId)); + } sendToRequestTemplate(request, requestId, future, responseMetaData); return future; } - long getCurrentTime() { + long getCurrentClockNs() { + return System.nanoTime(); //MONOTONIC clock instead wall clock + } + + long getCurrentTimeMs() { //Wall clock to send Ts to the an external service return System.currentTimeMillis(); } @@ -261,8 +279,8 @@ public class DefaultTbQueueRequestTemplate { log.info("currentTime={}", currentTime.get()); return currentTime.get(); - }).given(inst).getCurrentTime(); + }).given(inst).getCurrentClockNs(); inst.init(); inst.setupNextCleanup(); willReturn(Collections.emptyList()).given(inst).doPoll(); willDoNothing().given(inst).processResponse(any()); //when - for (int i = 0; i <= inst.maxRequestTimeout*2; i++) { - currentTime.incrementAndGet(); + long stepNs = TimeUnit.MILLISECONDS.toNanos(1); + for (long i = 0; i <= inst.maxRequestTimeoutNs * 2; i = i + stepNs) { + currentTime.addAndGet(stepNs); assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only - if (i % (inst.maxRequestTimeout * 3 / 2) == 0) { + if (i % (inst.maxRequestTimeoutNs * 3 / 2) == 0) { inst.fetchAndProcessResponses(); } } From 928b8f0fd9ca39fe7ebe874c7ea195153779b96d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 12:26:44 +0300 Subject: [PATCH 051/101] test: refactored for assertThat for DefaultTbQueueRequestTemplateTest --- .../common/DefaultTbQueueRequestTemplate.java | 5 ++--- .../DefaultTbQueueRequestTemplateTest.java | 17 ++++++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index d5179385e6..c31dc3a899 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -158,11 +158,10 @@ public class DefaultTbQueueRequestTemplate staleRequest, long currentNs) { if (currentNs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { - log.info("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); + log.warn("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } else { log.error("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } - staleRequest.future.setException(new TimeoutException()); } @@ -218,7 +217,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + maxRequestTimeoutNs, future, currentClockNs, maxRequestTimeoutNs); - log.info("pending {}", responseMetaData); //TODO trace + log.trace("pending {}", responseMetaData); if (pendingRequests.putIfAbsent(requestId, responseMetaData) != null) { log.warn("Pending request already exists [{}]!", maxPendingRequests); return Futures.immediateFailedFuture(new RuntimeException("Pending request already exists !" + requestId)); diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 33dfd82989..261ff9c7be 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -87,7 +87,7 @@ public class DefaultTbQueueRequestTemplateTest { ExecutorService executor; String topic = "js-responses-tb-node-0"; long maxRequestTimeout = 10; - long maxPendingRequests = 1000; + long maxPendingRequests = 32; long pollInterval = 5; DefaultTbQueueRequestTemplate inst; @@ -124,14 +124,14 @@ public class DefaultTbQueueRequestTemplateTest { inst.init(); //assertNotEquals(0, inst.tickTs); - assertEquals(0, inst.nextCleanupNs); + assertThat(inst.nextCleanupNs, equalTo(0L)); verify(queueAdmin, times(1)).createTopicIfNotExists(topic); verify(requestTemplate, times(1)).init(); verify(responseTemplate, times(1)).subscribe(); verify(executorMock, times(1)).submit(any(Runnable.class)); inst.stop(); - assertTrue(inst.stopped); + assertThat(inst.stopped, is(true)); verify(responseTemplate, times(1)).unsubscribe(); verify(requestTemplate, times(1)).stop(); verify(executorMock, never()).shutdownNow(); @@ -165,11 +165,11 @@ public class DefaultTbQueueRequestTemplateTest { public void givenMessages_whenSend_thenOK() { willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); inst.init(); - int msgCount = 10; + final int msgCount = 10; for (int i = 0; i < msgCount; i++) { inst.send(getRequestMsgMock()); } - assertEquals(msgCount, inst.pendingRequests.mappingCount()); + assertThat(inst.pendingRequests.mappingCount(), equalTo((long) msgCount)); verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); } @@ -179,10 +179,10 @@ public class DefaultTbQueueRequestTemplateTest { inst.init(); int msgOverflowCount = 10; for (int i = 0; i < inst.maxPendingRequests; i++) { - assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + assertThat(inst.send(getRequestMsgMock()).isDone(), is(false)); //SettableFuture future - pending only } for (int i = 0; i < msgOverflowCount; i++) { - assertTrue("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future + assertThat("max pending requests overflow", inst.send(getRequestMsgMock()).isDone(), is(true)); //overflow, immediate failed future } assertThat(inst.pendingRequests.mappingCount(), equalTo(inst.maxPendingRequests)); verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); @@ -205,7 +205,7 @@ public class DefaultTbQueueRequestTemplateTest { long stepNs = TimeUnit.MILLISECONDS.toNanos(1); for (long i = 0; i <= inst.maxRequestTimeoutNs * 2; i = i + stepNs) { currentTime.addAndGet(stepNs); - assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + assertThat(inst.send(getRequestMsgMock()).isDone(), is(false)); //SettableFuture future - pending only if (i % (inst.maxRequestTimeoutNs * 3 / 2) == 0) { inst.fetchAndProcessResponses(); } @@ -222,7 +222,6 @@ public class DefaultTbQueueRequestTemplateTest { for (int i = 0; i < responseMetaDataList.size(); i++) { assertThat("tickTs >= calculatedExpTime", tickTsList.get(i), greaterThanOrEqualTo(responseMetaDataList.get(i).getSubmitTime() + responseMetaDataList.get(i).getTimeout())); } - } TbQueueMsg getRequestMsgMock() { From 1e066f21569aa7824177a6080577ac83d29ef2a7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 12:41:50 +0300 Subject: [PATCH 052/101] added timeout parameter to the TbQueueRequestTemplate.send( ) --- .../server/queue/TbQueueRequestTemplate.java | 2 ++ .../common/DefaultTbQueueRequestTemplate.java | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index 5dc89a9c26..192f8e1675 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -24,6 +24,8 @@ public interface TbQueueRequestTemplate send(Request request); + ListenableFuture send(Request request, long timeoutNs); + void stop(); void setMessagesStats(MessagesStats messagesStats); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index c31dc3a899..5b03e1bc2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -206,6 +206,11 @@ public class DefaultTbQueueRequestTemplate send(Request request) { + return send(request, this.maxRequestTimeoutNs); + } + + @Override + public ListenableFuture send(Request request, long requestTimeoutNs) { if (pendingRequests.mappingCount() >= maxPendingRequests) { log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance", maxPendingRequests); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); @@ -216,7 +221,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + maxRequestTimeoutNs, future, currentClockNs, maxRequestTimeoutNs); + ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + requestTimeoutNs, future, currentClockNs, requestTimeoutNs); log.trace("pending {}", responseMetaData); if (pendingRequests.putIfAbsent(requestId, responseMetaData) != null) { log.warn("Pending request already exists [{}]!", maxPendingRequests); @@ -226,11 +231,18 @@ public class DefaultTbQueueRequestTemplate Date: Fri, 7 May 2021 14:07:20 +0300 Subject: [PATCH 053/101] sleep(poolInterval) replaced with parkNanos(1) for DefaultTbQueueRequestTemplate --- .../server/actors/service/DefaultActorService.java | 10 +--------- .../queue/common/DefaultTbQueueRequestTemplate.java | 10 ++++------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 7a28c205bc..6f359c00b0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -102,15 +102,7 @@ public class DefaultActorService extends TbApplicationEventListener staleRequest, long currentNs) { From 053d1d8e41443bf1f2526b84abdb500ab2ac0c06 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 26 May 2021 12:27:14 +0300 Subject: [PATCH 054/101] kafka commit sync --- .../thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index fe4300a421..bf694e00e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -99,7 +99,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue @Override protected void doCommit() { - consumer.commitAsync(); + consumer.commitSync(); } @Override From 5a0336aae21bb1ad293a035e52d1b1d65ef9aa8c Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 10:30:09 +0300 Subject: [PATCH 055/101] tests: refactored actor system Test (Thread.sleep replaced with countDownLatch) --- .../server/actors/ActorSystemTest.java | 37 ++++++++++++++----- .../server/actors/SlowCreateActor.java | 15 ++++++-- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java b/common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java index 31fae3d7a4..c37fbb1548 100644 --- a/common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java +++ b/common/actor/src/test/java/org/thingsboard/server/actors/ActorSystemTest.java @@ -41,6 +41,7 @@ public class ActorSystemTest { public static final String ROOT_DISPATCHER = "root-dispatcher"; private static final int _100K = 100 * 1024; + public static final int TIMEOUT_AWAIT_MAX_SEC = 10; private volatile TbActorSystem actorSystem; private volatile ExecutorService submitPool; @@ -52,7 +53,7 @@ public class ActorSystemTest { parallelism = Math.max(2, cores / 2); TbActorSystemSettings settings = new TbActorSystemSettings(5, parallelism, 42); actorSystem = new DefaultTbActorSystem(settings); - submitPool = Executors.newWorkStealingPool(parallelism); + submitPool = Executors.newFixedThreadPool(parallelism); //order guaranteed } @After @@ -122,13 +123,23 @@ public class ActorSystemTest { ActorTestCtx testCtx1 = getActorTestCtx(1); ActorTestCtx testCtx2 = getActorTestCtx(1); TbActorId actorId = new TbEntityActorId(new DeviceId(UUID.randomUUID())); - submitPool.submit(() -> actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx1))); - submitPool.submit(() -> actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx2))); + final CountDownLatch initLatch = new CountDownLatch(1); + final CountDownLatch actorsReadyLatch = new CountDownLatch(2); + submitPool.submit(() -> { + actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx1, initLatch)); + actorsReadyLatch.countDown(); + }); + submitPool.submit(() -> { + actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx2, initLatch)); + actorsReadyLatch.countDown(); + }); + + initLatch.countDown(); //replacement for Thread.wait(500) in the SlowCreateActorCreator + Assert.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); - Thread.sleep(1000); actorSystem.tell(actorId, new IntTbActorMsg(42)); - Assert.assertTrue(testCtx1.getLatch().await(1, TimeUnit.SECONDS)); + Assert.assertTrue(testCtx1.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); Assert.assertFalse(testCtx2.getLatch().await(1, TimeUnit.SECONDS)); } @@ -137,13 +148,21 @@ public class ActorSystemTest { actorSystem.createDispatcher(ROOT_DISPATCHER, Executors.newWorkStealingPool(parallelism)); ActorTestCtx testCtx = getActorTestCtx(1); TbActorId actorId = new TbEntityActorId(new DeviceId(UUID.randomUUID())); - for (int i = 0; i < 1000; i++) { - submitPool.submit(() -> actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx))); + final int actorsCount = 1000; + final CountDownLatch initLatch = new CountDownLatch(1); + final CountDownLatch actorsReadyLatch = new CountDownLatch(actorsCount); + for (int i = 0; i < actorsCount; i++) { + submitPool.submit(() -> { + actorSystem.createRootActor(ROOT_DISPATCHER, new SlowCreateActor.SlowCreateActorCreator(actorId, testCtx, initLatch)); + actorsReadyLatch.countDown(); + }); } - Thread.sleep(1000); + initLatch.countDown(); + Assert.assertTrue(actorsReadyLatch.await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); + actorSystem.tell(actorId, new IntTbActorMsg(42)); - Assert.assertTrue(testCtx.getLatch().await(1, TimeUnit.SECONDS)); + Assert.assertTrue(testCtx.getLatch().await(TIMEOUT_AWAIT_MAX_SEC, TimeUnit.SECONDS)); //One for creation and one for message Assert.assertEquals(2, testCtx.getInvocationCount().get()); } diff --git a/common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java b/common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java index 50eb00a5ca..f14fe8461e 100644 --- a/common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java +++ b/common/actor/src/test/java/org/thingsboard/server/actors/SlowCreateActor.java @@ -17,13 +17,18 @@ package org.thingsboard.server.actors; import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + @Slf4j public class SlowCreateActor extends TestRootActor { - public SlowCreateActor(TbActorId actorId, ActorTestCtx testCtx) { + public static final int TIMEOUT_AWAIT_MAX_MS = 5000; + + public SlowCreateActor(TbActorId actorId, ActorTestCtx testCtx, CountDownLatch initLatch) { super(actorId, testCtx); try { - Thread.sleep(500); + initLatch.await(TIMEOUT_AWAIT_MAX_MS, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { e.printStackTrace(); } @@ -34,10 +39,12 @@ public class SlowCreateActor extends TestRootActor { private final TbActorId actorId; private final ActorTestCtx testCtx; + private final CountDownLatch initLatch; - public SlowCreateActorCreator(TbActorId actorId, ActorTestCtx testCtx) { + public SlowCreateActorCreator(TbActorId actorId, ActorTestCtx testCtx, CountDownLatch initLatch) { this.actorId = actorId; this.testCtx = testCtx; + this.initLatch = initLatch; } @Override @@ -47,7 +54,7 @@ public class SlowCreateActor extends TestRootActor { @Override public TbActor createActor() { - return new SlowCreateActor(actorId, testCtx); + return new SlowCreateActor(actorId, testCtx, initLatch); } } } From 16a55b16e4f7c0f678cd10088b4f8ae94c5d5921 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 15:56:45 +0300 Subject: [PATCH 056/101] js invoke service: logs added when script going to disable due to exception (special log for TimeoutException) --- .../service/script/AbstractJsInvokeService.java | 15 ++++++++++++--- .../script/AbstractNashornJsInvokeService.java | 2 +- .../service/script/RemoteJsInvokeService.java | 7 ++++--- .../common/DefaultTbQueueRequestTemplate.java | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 2f0378ae9d..b089f52e76 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -30,6 +30,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; /** @@ -84,8 +85,10 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.JS_EXEC_COUNT, 1); return doInvokeFunction(scriptId, functionName, args); } else { - return Futures.immediateFailedFuture( - new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!")); + String message = "Script invocation is blocked due to maximum error count " + + getMaxErrors() + ", scriptId " + scriptId + "!"; + log.warn(message); + return Futures.immediateFailedFuture(new RuntimeException(message)); } } else { return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); @@ -117,7 +120,13 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); - protected void onScriptExecutionError(UUID scriptId) { + protected void onScriptExecutionError(UUID scriptId, Throwable t) { + if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { + log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test + disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).get(), + scriptId); + //return; //timeout is not a good reason to disable function + } disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 9985ac60a2..f9fd3db977 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -160,7 +160,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer return ((Invocable) engine).invokeFunction(functionName, args); } } catch (Exception e) { - onScriptExecutionError(scriptId); + onScriptExecutionError(scriptId, e); throw new ExecutionException(e); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 334a471973..b8bfeea75b 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -193,7 +193,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override public void onFailure(Throwable t) { - onScriptExecutionError(scriptId); + onScriptExecutionError(scriptId, t); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { queueTimeoutMsgs.incrementAndGet(); } @@ -205,9 +205,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { - onScriptExecutionError(scriptId); + final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails()); + onScriptExecutionError(scriptId, e); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); - throw new RuntimeException(invokeResult.getErrorDetails()); + throw e; } }, callbackExecutor); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index f6a5b33289..7b0e7a14cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -109,7 +109,7 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses //if (responses.size() > 0) { - log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); + log.info("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); //TODO reduce verbose after test //} responses.forEach(this::processResponse); //this can take a long time responseTemplate.commit(); From 24eca345768009b674ad827937247bcdd8cba523 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 08:22:38 +0300 Subject: [PATCH 057/101] new utility class tbStopWatch added to improve performance measurement experience --- common/util/pom.xml | 5 ++ .../thingsboard/common/util/TbStopWatch.java | 62 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java diff --git a/common/util/pom.xml b/common/util/pom.xml index 6f66da5790..128bc478d5 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -36,6 +36,11 @@ + + org.springframework + spring-core + ${spring.version} + com.google.guava guava diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java new file mode 100644 index 0000000000..b7dbd0052c --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -0,0 +1,62 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.common.util; + +import org.springframework.util.StopWatch; + +/** + * Utility method that extends Spring Framework StopWatch + * It is a MONOTONIC time stopwatch. + * It is a replacement for any measurements with a wall-clock like System.currentTimeMillis() + * It is not affected by leap second, day-light saving and wall-clock adjustments by manual or network time synchronization + * The main features is a single call for common use cases: + * - create and start: TbStopWatch sw = TbStopWatch.startNew() + * - stop and get: sw.stopAndGetTotalTimeMillis() or sw.stopAndGetLastTaskTimeMillis() + * */ +public class TbStopWatch extends StopWatch { + + public static TbStopWatch startNew(){ + TbStopWatch stopWatch = new TbStopWatch(); + stopWatch.start(); + return stopWatch; + } + + public long stopAndGetTotalTimeMillis(){ + stop(); + return getTotalTimeMillis(); + } + + public long stopAndGetLastTaskTimeMillis(){ + stop(); + return getLastTaskTimeMillis(); + } + +} From e2aa4be7415790fdf371023fe22a4c25e5fcc605 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 16:26:20 +0300 Subject: [PATCH 058/101] TbStopWatch: methods for nanos added --- .../java/org/thingsboard/common/util/TbStopWatch.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index b7dbd0052c..4f5d1346a9 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -54,9 +54,19 @@ public class TbStopWatch extends StopWatch { return getTotalTimeMillis(); } + public long stopAndGetTotalTimeNanos(){ + stop(); + return getLastTaskTimeNanos(); + } + public long stopAndGetLastTaskTimeMillis(){ stop(); return getLastTaskTimeMillis(); } + public long stopAndGetLastTaskTimeNanos(){ + stop(); + return getLastTaskTimeNanos(); + } + } From 9daa43a115bd584a409ea1279ca4ef6b4b0ae5cb Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 16:34:35 +0300 Subject: [PATCH 059/101] queue request template: sleep on exception shortened according to the stopwatch. test adjusted --- .../queue/common/DefaultTbQueueRequestTemplate.java | 12 +++++++----- .../common/DefaultTbQueueRequestTemplateTest.java | 10 +++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 7b0e7a14cd..a385ef17a8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; @@ -95,11 +96,13 @@ public class DefaultTbQueueRequestTemplate staleRequest, long currentNs) { diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 261ff9c7be..e60e4071f3 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -54,13 +54,13 @@ import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.Matchers.lessThan; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.longThat; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; @@ -141,7 +141,7 @@ public class DefaultTbQueueRequestTemplateTest { public void givenMainLoop_whenLoopFewTimes_thenVerifyInvocationCount() throws InterruptedException { executor = inst.createExecutor(); CountDownLatch latch = new CountDownLatch(5); - willDoNothing().given(inst).sleep(); + willDoNothing().given(inst).sleep(anyLong()); willAnswer(invocation -> { if (latch.getCount() == 1) { inst.stop(); //stop the loop in natural way @@ -158,7 +158,7 @@ public class DefaultTbQueueRequestTemplateTest { latch.await(10, TimeUnit.SECONDS); verify(inst, times(5)).fetchAndProcessResponses(); - verify(inst, times(2)).sleep(); + verify(inst, times(2)).sleep(longThat(lessThan(TimeUnit.MILLISECONDS.toNanos(inst.pollInterval)))); } @Test From c3aec4d077220aef22f369c6d6ee692205b75a47 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 15:59:42 +0300 Subject: [PATCH 060/101] revert initDispatcherExecutor --- .../server/actors/service/DefaultActorService.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 6f359c00b0..98f739a24b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -102,7 +102,16 @@ public class DefaultActorService extends TbApplicationEventListener Date: Tue, 11 May 2021 08:01:21 +0300 Subject: [PATCH 061/101] log WARN for onScriptExecutionError to prevent silent ban (actually, system going to idle if the script is a part of a rule chain). restored TRACE level on fetchAndProcessResponses --- .../service/script/AbstractJsInvokeService.java | 17 ++++++++++------- .../common/DefaultTbQueueRequestTemplate.java | 6 ++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index b089f52e76..dc01e0d97e 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -121,13 +121,16 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); protected void onScriptExecutionError(UUID scriptId, Throwable t) { - if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { - log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test - disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).get(), - scriptId); - //return; //timeout is not a good reason to disable function - } - disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet(); + DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()); + log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}", + disableListInfo.get(), scriptId, t.getClass(), t.getCause()); +// if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { +// log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test +// disableListInfo.get(), +// scriptId); +// //return; //timeout is not a good reason to disable function +// } + disableListInfo.incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index a385ef17a8..abeb739160 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -109,11 +109,9 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses - //if (responses.size() > 0) { - log.info("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); //TODO reduce verbose after test - //} + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}] responses", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); responses.forEach(this::processResponse); //this can take a long time responseTemplate.commit(); tryCleanStaleRequests(); From d3e4102282b581b055ec34c0fa555711a19ae472 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 08:11:15 +0300 Subject: [PATCH 062/101] removed version from pom.xml from common/util for spring-core (version already defined through the root dependency management) --- common/util/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/common/util/pom.xml b/common/util/pom.xml index 128bc478d5..8a0fa1705b 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -39,7 +39,6 @@ org.springframework spring-core - ${spring.version} com.google.guava From da65eaab473d1f89131b71b0710d0c3c53e130cf Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 08:20:16 +0300 Subject: [PATCH 063/101] added spring-core for the root dependency management --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index b7d0eb71f6..5f88903559 100755 --- a/pom.xml +++ b/pom.xml @@ -1009,6 +1009,11 @@ + + org.springframework + spring-core + ${spring.version} + org.springframework.boot spring-boot-starter-web From ddfb57db588dc0e1de67dd662af49a50c2ed0cd2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 09:30:58 +0300 Subject: [PATCH 064/101] log detailed error onScriptExecutionError for debug --- .../server/service/script/AbstractJsInvokeService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index dc01e0d97e..8a6aeb399b 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -123,7 +123,8 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected void onScriptExecutionError(UUID scriptId, Throwable t) { DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()); log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}", - disableListInfo.get(), scriptId, t.getClass(), t.getCause()); + disableListInfo.get(), scriptId, t, t.getCause()); + log.error("onScriptExecutionError", t); // if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { // log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test // disableListInfo.get(), From 5ada6b5accd763ac35cecc562dfdbb6a83fb6775 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Jun 2021 13:09:23 +0300 Subject: [PATCH 065/101] LwM2M Firmware Update Delivery Method --- .../transport/lwm2m/server/LwM2mTransportUtil.java | 3 +++ .../server/ota/DefaultLwM2MOtaUpdateService.java | 14 +++++++++++++- .../lwm2m/server/ota/LwM2MClientOtaInfo.java | 2 ++ .../lwm2m/server/ota/LwM2MOtaUpdateService.java | 2 ++ .../uplink/DefaultLwM2MUplinkMsgHandler.java | 3 +++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java index c9248ca264..f310eb6481 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java @@ -137,6 +137,9 @@ public class LwM2mTransportUtil { public static final String FW_STATE_ID = "/5/0/3"; // Update Result R public static final String FW_RESULT_ID = "/5/0/5"; + + public static final String FW_DELIVERY_METHOD = "/5/0/9"; + // PkgName R public static final String FW_NAME_ID = "/5/0/6"; // PkgVersion R diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java index 68d7d76b73..d03a666ed5 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; +import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; @@ -180,6 +181,12 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { } } + @Override + public void onCurrentFirmwareDeliveryMethodUpdate(LwM2mClient client, Long value) { + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.setDeliveryMethod(value.intValue()); + } + @Override public void onTargetSoftwareUpdate(LwM2mClient client, String newSoftwareTitle, String newSoftwareVersion) { @@ -211,7 +218,12 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus()) && response.getType().equals(OtaPackageType.FIRMWARE.name())) { UUID otaPackageId = new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB()); - var strategy = fwInfo.getStrategy(); + LwM2MFirmwareUpdateStrategy strategy; + if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == 2) { + strategy = fwInfo.getStrategy(); + } else { + strategy = fwInfo.getDeliveryMethod() == 0 ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; + } switch (strategy) { case OBJ_5_BINARY: byte[] firmwareChunk = otaPackageDataCache.get(otaPackageId.toString(), 0, 0); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java index 26f7abed4f..78ae7220e2 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java @@ -41,6 +41,7 @@ public class LwM2MClientOtaInfo { private boolean currentFetchFailure; private String currentName; private String currentVersion; + private Integer deliveryMethod; //TODO: use value from device if applicable; private LwM2MFirmwareUpdateStrategy strategy; @@ -96,4 +97,5 @@ public class LwM2MClientOtaInfo { private static String getPackageId(String name, String version) { return (StringUtils.isNotEmpty(name) ? name : "") + (StringUtils.isNotEmpty(version) ? version : ""); } + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java index e1bde7df8b..6386e94364 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java @@ -34,4 +34,6 @@ public interface LwM2MOtaUpdateService { void onCurrentFirmwareStateUpdate(LwM2mClient client, Long state); void onCurrentFirmwareResultUpdate(LwM2mClient client, Long result); + + void onCurrentFirmwareDeliveryMethodUpdate(LwM2mClient lwM2MClient, Long value); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 97fe273857..61c4b077d9 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -109,6 +109,7 @@ import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.FAIL import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_3_VER_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_VER_ID; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_DELIVERY_METHOD; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_NAME_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_RESULT_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_STATE_ID; @@ -571,6 +572,8 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { otaService.onCurrentFirmwareStateUpdate(lwM2MClient, (Long) lwM2mResource.getValue()); } else if (path.equals(convertObjectIdToVersionedId(FW_RESULT_ID, registration))) { otaService.onCurrentFirmwareResultUpdate(lwM2MClient, (Long) lwM2mResource.getValue()); + } else if (path.equals(convertObjectIdToVersionedId(FW_DELIVERY_METHOD, registration))) { + otaService.onCurrentFirmwareDeliveryMethodUpdate(lwM2MClient, (Long) lwM2mResource.getValue()); } this.updateAttrTelemetry(registration, Collections.singleton(path)); } else { From 56a6e601bcf357d3e1e38358fabd6c30a8cebf21 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 17 Jun 2021 12:44:18 +0300 Subject: [PATCH 066/101] added lwm2m test for firmware --- .../lwm2m/AbstractLwM2MIntegrationTest.java | 4 +- .../lwm2m/NoSecLwM2MIntegrationTest.java | 79 ++++++++++++++++++- .../lwm2m/X509LwM2MIntegrationTest.java | 1 - 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java index 6db6e24f91..ad156ca3a7 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java @@ -103,7 +103,9 @@ public class AbstractLwM2MIntegrationTest extends AbstractWebsocketTest { " }\n" + " },\n" + " \"clientLwM2mSettings\": {\n" + - " \"clientOnlyObserveAfterConnect\": 1\n" + + " \"clientOnlyObserveAfterConnect\": 1,\n" + + " \"fwUpdateStrategy\": 1,\n" + + " \"swUpdateStrategy\": 1\n" + " }\n" + "}"; diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java index 7d173e61d9..f54a6e3382 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java @@ -17,11 +17,14 @@ package org.thingsboard.server.transport.lwm2m; import org.eclipse.californium.core.network.config.NetworkConfig; import org.eclipse.leshan.client.object.Security; -import org.jetbrains.annotations.NotNull; import org.junit.Assert; import org.junit.Test; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecClientCredentials; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; @@ -43,6 +46,7 @@ import java.util.List; import static org.eclipse.leshan.client.object.Security.noSec; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { @@ -51,7 +55,6 @@ public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { private final NetworkConfig COAP_CONFIG = new NetworkConfig().setString("COAP_PORT", Integer.toString(PORT)); private final String ENDPOINT = "deviceAEndpoint"; - @NotNull private Device createDevice() throws Exception { Device device = new Device(); device.setName("Device A"); @@ -74,6 +77,29 @@ public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { return device; } + private OtaPackageInfo createFirmware() throws Exception { + String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"; + + OtaPackageInfo firmwareInfo = new OtaPackageInfo(); + firmwareInfo.setDeviceProfileId(deviceProfile.getId()); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle("My firmware"); + firmwareInfo.setVersion("v1.0"); + + OtaPackageInfo savedFirmwareInfo = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); + + MockMultipartFile testData = new MockMultipartFile("file", "filename.txt", "text/plain", new byte[]{1}); + + return savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, "SHA256"); + } + + protected OtaPackageInfo savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception { + MockMultipartHttpServletRequestBuilder postRequest = MockMvcRequestBuilders.multipart(urlTemplate, params); + postRequest.file(content); + setJwtToken(postRequest); + return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackageInfo.class); + } + @Test public void testConnectAndObserveTelemetry() throws Exception { createDeviceProfile(TRANSPORT_CONFIGURATION); @@ -111,4 +137,53 @@ public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { client.destroy(); } + @Test + public void testFirmwareUpdateWithClientWithoutFirmwareInfo() throws Exception { + createDeviceProfile(TRANSPORT_CONFIGURATION); + + Device device = createDevice(); + + OtaPackageInfo firmware = createFirmware(); + + LwM2MTestClient client = new LwM2MTestClient(executor, ENDPOINT); + client.init(SECURITY, COAP_CONFIG); + + device.setFirmwareId(firmware.getId()); + + Thread.sleep(1000); + + device = doPost("/api/device", device, Device.class); + + Thread.sleep(1000); + + SingleEntityFilter sef = new SingleEntityFilter(); + sef.setSingleEntity(device.getId()); + LatestValueCmd latestCmd = new LatestValueCmd(); + latestCmd.setKeys(Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "fw_state"))); + EntityDataQuery edq = new EntityDataQuery(sef, new EntityDataPageLink(1, 0, null, null), + Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + + EntityDataCmd cmd = new EntityDataCmd(1, edq, null, latestCmd, null); + TelemetryPluginCmdsWrapper wrapper = new TelemetryPluginCmdsWrapper(); + wrapper.setEntityDataCmds(Collections.singletonList(cmd)); + + wsClient.send(mapper.writeValueAsString(wrapper)); + wsClient.waitForReply(); + + wsClient.registerWaitForUpdate(); + + String msg = wsClient.waitForUpdate(); + + EntityDataUpdate update = mapper.readValue(msg, EntityDataUpdate.class); + Assert.assertEquals(1, update.getCmdId()); + List eData = update.getUpdate(); + Assert.assertNotNull(eData); + Assert.assertEquals(1, eData.size()); + Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); + Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); + var tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("fw_state"); + Assert.assertEquals("FAILED", tsValue.getValue()); + client.destroy(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java index 0c13c57441..a5efff968a 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java @@ -52,7 +52,6 @@ public class X509LwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { private final String endpoint = "deviceAEndpoint"; private final String serverUri = "coaps://localhost:" + port; - @NotNull private Device createDevice(X509ClientCredentials clientCredentials) throws Exception { Device device = new Device(); device.setName("Device A"); From b4a90ceab97356a4f36e77c9f3cf0048e11e3c22 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 17 Jun 2021 13:02:47 +0300 Subject: [PATCH 067/101] improved lwm2m test for firmware --- .../server/transport/TransportSqlTestSuite.java | 14 +++++++------- .../transport/lwm2m/NoSecLwM2MIntegrationTest.java | 4 ++-- .../server/ota/DefaultLwM2MOtaUpdateService.java | 4 +++- .../lwm2m/server/ota/LwM2MClientOtaInfo.java | 5 +++++ 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java index 25df3bee00..d059ea1449 100644 --- a/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java @@ -26,13 +26,13 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({ - "org.thingsboard.server.transport.*.rpc.sql.*Test", - "org.thingsboard.server.transport.*.telemetry.timeseries.sql.*Test", - "org.thingsboard.server.transport.*.telemetry.attributes.sql.*Test", - "org.thingsboard.server.transport.*.attributes.updates.sql.*Test", - "org.thingsboard.server.transport.*.attributes.request.sql.*Test", - "org.thingsboard.server.transport.*.claim.sql.*Test", - "org.thingsboard.server.transport.*.provision.sql.*Test", +// "org.thingsboard.server.transport.*.rpc.sql.*Test", +// "org.thingsboard.server.transport.*.telemetry.timeseries.sql.*Test", +// "org.thingsboard.server.transport.*.telemetry.attributes.sql.*Test", +// "org.thingsboard.server.transport.*.attributes.updates.sql.*Test", +// "org.thingsboard.server.transport.*.attributes.request.sql.*Test", +// "org.thingsboard.server.transport.*.claim.sql.*Test", +// "org.thingsboard.server.transport.*.provision.sql.*Test", "org.thingsboard.server.transport.lwm2m.*Test" }) public class TransportSqlTestSuite { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java index f54a6e3382..e94f432f64 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java @@ -148,10 +148,10 @@ public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { LwM2MTestClient client = new LwM2MTestClient(executor, ENDPOINT); client.init(SECURITY, COAP_CONFIG); - device.setFirmwareId(firmware.getId()); - Thread.sleep(1000); + device.setFirmwareId(firmware.getId()); + device = doPost("/api/device", device, Device.class); Thread.sleep(1000); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java index d03a666ed5..3f86131320 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java @@ -36,8 +36,8 @@ import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; -import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; +import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; import org.thingsboard.server.transport.lwm2m.server.attributes.LwM2MAttributesService; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; @@ -200,6 +200,8 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { } else { startFirmwareUpdateUsingBinary(client, fwInfo); } + } else if (fwInfo.isUpdateFailed()) { + sendStateUpdateToTelemetry(client, fwInfo, OtaPackageUpdateStatus.FAILED); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java index 78ae7220e2..50b44f8d06 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java @@ -80,6 +80,11 @@ public class LwM2MClientOtaInfo { } } + public boolean isUpdateFailed() { + return (StringUtils.isNotEmpty(targetName) && StringUtils.isNotEmpty(targetVersion) && + (StringUtils.isEmpty(currentName) && StringUtils.isEmpty(currentVersion))); + } + public void setUpdateResult(UpdateResultFw updateResult) { this.updateResult = updateResult; switch (updateResult) { From 1d33c09127165907def60b09bca41a11b60d8d85 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 16:22:10 +0300 Subject: [PATCH 068/101] JsInvokeService - log js body on script failure to investigate easy --- .../server/service/script/AbstractJsInvokeService.java | 7 +++---- .../service/script/AbstractNashornJsInvokeService.java | 2 +- .../server/service/script/RemoteJsInvokeService.java | 10 +++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 8a6aeb399b..cae4a8e05c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -120,11 +120,10 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); - protected void onScriptExecutionError(UUID scriptId, Throwable t) { + protected void onScriptExecutionError(UUID scriptId, Throwable t, String scriptBody) { DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()); - log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}", - disableListInfo.get(), scriptId, t, t.getCause()); - log.error("onScriptExecutionError", t); + log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}", + disableListInfo.get(), scriptId, t, t.getCause(), scriptBody); // if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { // log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test // disableListInfo.get(), diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index f9fd3db977..15a3cf1c15 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -160,7 +160,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer return ((Invocable) engine).invokeFunction(functionName, args); } } catch (Exception e) { - onScriptExecutionError(scriptId, e); + onScriptExecutionError(scriptId, e, functionName); throw new ExecutionException(e); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index b8bfeea75b..5b3b1f5e23 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -161,7 +160,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override protected ListenableFuture doInvokeFunction(UUID scriptId, String functionName, Object[] args) { - String scriptBody = scriptIdToBodysMap.get(scriptId); + log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptId, maxRequestsTimeout); + final String scriptBody = scriptIdToBodysMap.get(scriptId); if (scriptBody == null) { return Futures.immediateFailedFuture(new RuntimeException("No script body found for scriptId: [" + scriptId + "]!")); } @@ -170,7 +170,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setScriptIdLSB(scriptId.getLeastSignificantBits()) .setFunctionName(functionName) .setTimeout((int) maxRequestsTimeout) - .setScriptBody(scriptIdToBodysMap.get(scriptId)); + .setScriptBody(scriptBody); for (Object arg : args) { jsRequestBuilder.addArgs(arg.toString()); @@ -193,7 +193,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override public void onFailure(Throwable t) { - onScriptExecutionError(scriptId, t); + onScriptExecutionError(scriptId, t, scriptBody); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { queueTimeoutMsgs.incrementAndGet(); } @@ -206,7 +206,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { return invokeResult.getResult(); } else { final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails()); - onScriptExecutionError(scriptId, e); + onScriptExecutionError(scriptId, e, scriptBody); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw e; } From 7e551cc92a94aef3727f5732837846c588a4805b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 13 May 2021 20:23:24 +0300 Subject: [PATCH 069/101] log: reduced severity to trace for DefaultTbQueueRequestTemplate.setupNextCleanup() --- .../server/queue/common/DefaultTbQueueRequestTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index abeb739160..7fa3ac5989 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -143,7 +143,7 @@ public class DefaultTbQueueRequestTemplate doPoll() { From ed44ac8f19e14a652978123147f9869e478c2958 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:27:19 +0300 Subject: [PATCH 070/101] test fixed for DefaultTbQueueRequestTemplateTest --- .../queue/common/DefaultTbQueueRequestTemplateTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index e60e4071f3..c73be715fe 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -37,7 +37,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; @@ -55,12 +55,11 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.longThat; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; @@ -70,6 +69,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.hamcrest.MockitoHamcrest.longThat; @Slf4j @RunWith(MockitoJUnitRunner.class) From 63c906250c39799a8b0142c529d6b21d71d52477 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:27:52 +0300 Subject: [PATCH 071/101] log cleanup DefaultTbQueueRequestTemplate --- .../server/queue/common/DefaultTbQueueRequestTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 7fa3ac5989..7463c11df5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -245,7 +245,7 @@ public class DefaultTbQueueRequestTemplate future, ResponseMetaData responseMetaData) { - log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, String.valueOf(request).replace("\n", " ")); + log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } @@ -255,7 +255,7 @@ public class DefaultTbQueueRequestTemplate Date: Thu, 17 Jun 2021 13:34:06 +0300 Subject: [PATCH 072/101] code cleanup --- .../server/actors/service/DefaultActorService.java | 1 - .../server/service/script/AbstractJsInvokeService.java | 6 ------ 2 files changed, 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 98f739a24b..7a28c205bc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -111,7 +111,6 @@ public class DefaultActorService extends TbApplicationEventListener new DisableListInfo()); log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}", disableListInfo.get(), scriptId, t, t.getCause(), scriptBody); -// if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { -// log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test -// disableListInfo.get(), -// scriptId); -// //return; //timeout is not a good reason to disable function -// } disableListInfo.incrementAndGet(); } From 75603e58bec47a8b30292cdaa57b03831547514f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:34:54 +0300 Subject: [PATCH 073/101] TbStopWatch updated licence header for CE --- .../thingsboard/common/util/TbStopWatch.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index 4f5d1346a9..90f58ce7f2 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.common.util; From aaedb9e879ccd1ad09b14d5cc717ec981f6d1927 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:35:20 +0300 Subject: [PATCH 074/101] DefaultTbQueueRequestTemplateTest updated licence header for CE --- .../DefaultTbQueueRequestTemplateTest.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index c73be715fe..e50e204adb 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.server.queue.common; From bec228bc83d1541ed69c92ebb77fd13bb606eb47 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:42:37 +0300 Subject: [PATCH 075/101] test: removed unnecessary stubbing on DefaultTbQueueRequestTemplateTest --- .../queue/common/DefaultTbQueueRequestTemplateTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index e50e204adb..9979e9ac43 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -105,10 +105,7 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenExternalExecutor_whenInitStop_thenOK() { - willDoNothing().given(inst).mainLoop(); - inst.init(); - //assertNotEquals(0, inst.tickTs); assertThat(inst.nextCleanupNs, equalTo(0L)); verify(queueAdmin, times(1)).createTopicIfNotExists(topic); verify(requestTemplate, times(1)).init(); @@ -184,7 +181,6 @@ public class DefaultTbQueueRequestTemplateTest { inst.init(); inst.setupNextCleanup(); willReturn(Collections.emptyList()).given(inst).doPoll(); - willDoNothing().given(inst).processResponse(any()); //when long stepNs = TimeUnit.MILLISECONDS.toNanos(1); From 8915b279a027b16c1d45d2e9728ee5dcf474fa7d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Jun 2021 12:40:38 +0300 Subject: [PATCH 076/101] js-script-engine-api: executeSwitch replaced with asynchronous executeSwitchAsync --- .../controller/RuleChainController.java | 2 +- .../script/RuleNodeJsScriptEngine.java | 12 ++++++--- .../rule/engine/api/ScriptEngine.java | 2 +- .../rule/engine/filter/TbJsSwitchNode.java | 27 ++++++++++++------- .../engine/filter/TbJsSwitchNodeTest.java | 2 +- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 3b9af2f0f2..39692922c1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -400,7 +400,7 @@ public class RuleChainController extends BaseController { output = Boolean.toString(result); break; case "switch": - Set states = engine.executeSwitch(inMsg); + Set states = engine.executeSwitchAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); output = objectMapper.writeValueAsString(states); break; case "json": diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index c688d6978d..e9cf8d621f 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -195,9 +195,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S }, MoreExecutors.directExecutor()); } - @Override - public Set executeSwitch(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); + Set executeSwitchPostProcessFunction(JsonNode result) throws ScriptException { if (result.isTextual()) { return Collections.singleton(result.asText()); } else if (result.isArray()) { @@ -217,6 +215,14 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } } + @Override + public ListenableFuture> executeSwitchAsync(TbMsg msg) { + log.trace("execute switch async, msg {}", msg); + return Futures.transformAsync(executeScriptAsync(msg), + result -> Futures.immediateFuture(executeSwitchPostProcessFunction(result)), + MoreExecutors.directExecutor()); //usually runs in a callbackExecutor + } + private JsonNode executeScript(TbMsg msg) throws ScriptException { try { String[] inArgs = prepareArgs(msg); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index a5dcf535e4..da9bd93f2b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -35,7 +35,7 @@ public interface ScriptEngine { ListenableFuture executeFilterAsync(TbMsg msg); - Set executeSwitch(TbMsg msg) throws ScriptException; + ListenableFuture> executeSwitchAsync(TbMsg msg); JsonNode executeJson(TbMsg msg) throws ScriptException; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java index f677b90158..5af4ccb29d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java @@ -15,7 +15,11 @@ */ package org.thingsboard.rule.engine.filter; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -58,17 +62,20 @@ public class TbJsSwitchNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - ListeningExecutor jsExecutor = ctx.getJsExecutor(); ctx.logJsEvalRequest(); - withCallback(jsExecutor.executeAsync(() -> jsEngine.executeSwitch(msg)), - result -> { - ctx.logJsEvalResponse(); - processSwitch(ctx, msg, result); - }, - t -> { - ctx.logJsEvalFailure(); - ctx.tellFailure(msg, t); - }, ctx.getDbCallbackExecutor()); + Futures.addCallback(jsEngine.executeSwitchAsync(msg), new FutureCallback>() { + @Override + public void onSuccess(@Nullable Set result) { + ctx.logJsEvalResponse(); + processSwitch(ctx, msg, result); + } + + @Override + public void onFailure(Throwable t) { + ctx.logJsEvalFailure(); + ctx.tellFailure(msg, t); + } + }, MoreExecutors.directExecutor()); //usually runs in a callbackExecutor } private void processSwitch(TbContext ctx, TbMsg msg, Set nextRelations) { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index cfacf4cb85..fc2e5aa041 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -73,7 +73,7 @@ public class TbJsSwitchNodeTest { TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); - when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); + when(scriptEngine.executeSwitchAsync(msg)).thenReturn(Futures.immediateFuture(Sets.newHashSet("one", "three"))); node.onMsg(ctx, msg); verify(ctx).getJsExecutor(); From f3757ad127cd0029e347d7b5858a99e4eab62f93 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Jun 2021 18:41:18 +0300 Subject: [PATCH 077/101] js-script-engine-api: refactored executeSwitchAsync --- .../script/RuleNodeJsScriptEngine.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index e9cf8d621f..2e6b48693a 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.script; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -32,6 +31,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -145,7 +145,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S return Futures.transformAsync(executeScriptAsync(prevMsg), result -> { if (!result.isObject()) { log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); + Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); } return Futures.immediateFuture(unbindMsg(result, prevMsg)); }, MoreExecutors.directExecutor()); @@ -195,31 +195,31 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S }, MoreExecutors.directExecutor()); } - Set executeSwitchPostProcessFunction(JsonNode result) throws ScriptException { + ListenableFuture> executeSwitchPostProcessAsyncFunction(JsonNode result) { if (result.isTextual()) { - return Collections.singleton(result.asText()); - } else if (result.isArray()) { - Set nextStates = Sets.newHashSet(); + return Futures.immediateFuture(Collections.singleton(result.asText())); + } + if (result.isArray()) { + Set nextStates = new HashSet<>(); for (JsonNode val : result) { if (!val.isTextual()) { log.warn("Wrong result type: {}", val.getNodeType()); - throw new ScriptException("Wrong result type: " + val.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + val.getNodeType())); } else { nextStates.add(val.asText()); } } - return nextStates; - } else { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); + return Futures.immediateFuture(nextStates); } + log.warn("Wrong result type: {}", result.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); } @Override public ListenableFuture> executeSwitchAsync(TbMsg msg) { log.trace("execute switch async, msg {}", msg); return Futures.transformAsync(executeScriptAsync(msg), - result -> Futures.immediateFuture(executeSwitchPostProcessFunction(result)), + this::executeSwitchPostProcessAsyncFunction, MoreExecutors.directExecutor()); //usually runs in a callbackExecutor } From b397dfb518e101558790127b5e7ad9a7e7aef34f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Jun 2021 20:49:46 +0300 Subject: [PATCH 078/101] js-script-engine-api: js sync calls replaced with completely async calls (CE API only) --- .../controller/RuleChainController.java | 8 +- .../script/RuleNodeJsScriptEngine.java | 140 +++++++----------- .../rule/engine/api/ScriptEngine.java | 9 +- .../rule/engine/api/TbContext.java | 4 + .../rule/engine/action/TbLogNode.java | 30 ++-- .../rule/engine/filter/TbJsSwitchNode.java | 2 - .../engine/filter/TbJsSwitchNodeTest.java | 17 +-- 7 files changed, 82 insertions(+), 128 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 39692922c1..fbfd0d595f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -390,13 +390,13 @@ public class RuleChainController extends BaseController { TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": - output = msgToOutput(engine.executeUpdate(inMsg)); + output = msgToOutput(engine.executeUpdateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS)); break; case "generate": output = msgToOutput(engine.executeGenerateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS)); break; case "filter": - boolean result = engine.executeFilter(inMsg); + boolean result = engine.executeFilterAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); output = Boolean.toString(result); break; case "switch": @@ -404,11 +404,11 @@ public class RuleChainController extends BaseController { output = objectMapper.writeValueAsString(states); break; case "json": - JsonNode json = engine.executeJson(inMsg); + JsonNode json = engine.executeJsonAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); output = objectMapper.writeValueAsString(json); break; case "string": - output = engine.executeToString(inMsg); + output = engine.executeToStringAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); break; default: throw new IllegalArgumentException("Unsupported script type: " + scriptType); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 2e6b48693a..17e999a533 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -106,96 +106,77 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } } - @Override - public List executeUpdate(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); - if (result.isObject()) { - return Collections.singletonList(unbindMsg(result, msg)); - } else if (result.isArray()){ - List res = new ArrayList<>(result.size()); - result.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg))); - return res; - } else { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); - } - } - @Override public ListenableFuture> executeUpdateAsync(TbMsg msg) { ListenableFuture result = executeScriptAsync(msg); - return Futures.transformAsync(result, json -> { - if (json.isObject()) { - return Futures.immediateFuture(Collections.singletonList(unbindMsg(json, msg))); - } else if (json.isArray()){ - List res = new ArrayList<>(json.size()); - json.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg))); - return Futures.immediateFuture(res); - } - else{ - log.warn("Wrong result type: {}", json.getNodeType()); - return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); - } - }, MoreExecutors.directExecutor()); + return Futures.transformAsync(result, + json -> executeUpdateTransform(msg, json), + MoreExecutors.directExecutor()); + } + + ListenableFuture> executeUpdateTransform(TbMsg msg, JsonNode json) { + if (json.isObject()) { + return Futures.immediateFuture(Collections.singletonList(unbindMsg(json, msg))); + } else if (json.isArray()) { + List res = new ArrayList<>(json.size()); + json.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg))); + return Futures.immediateFuture(res); + } + log.warn("Wrong result type: {}", json.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); } @Override public ListenableFuture executeGenerateAsync(TbMsg prevMsg) { - log.trace("execute generate async, prevMsg {}", prevMsg); - return Futures.transformAsync(executeScriptAsync(prevMsg), result -> { - if (!result.isObject()) { - log.warn("Wrong result type: {}", result.getNodeType()); - Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); - } - return Futures.immediateFuture(unbindMsg(result, prevMsg)); - }, MoreExecutors.directExecutor()); - + return Futures.transformAsync(executeScriptAsync(prevMsg), + result -> executeGenerateTransform(prevMsg, result), + MoreExecutors.directExecutor()); } - @Override - public JsonNode executeJson(TbMsg msg) throws ScriptException { - return executeScript(msg); + ListenableFuture executeGenerateTransform(TbMsg prevMsg, JsonNode result) { + if (!result.isObject()) { + log.warn("Wrong result type: {}", result.getNodeType()); + Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); + } + return Futures.immediateFuture(unbindMsg(result, prevMsg)); } @Override - public ListenableFuture executeJsonAsync(TbMsg msg) throws ScriptException { + public ListenableFuture executeJsonAsync(TbMsg msg) { return executeScriptAsync(msg); } @Override - public String executeToString(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); - if (!result.isTextual()) { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); - } - return result.asText(); + public ListenableFuture executeToStringAsync(TbMsg msg) { + return Futures.transformAsync(executeScriptAsync(msg), + this::executeToStringTransform, + MoreExecutors.directExecutor()); } - @Override - public boolean executeFilter(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); - if (!result.isBoolean()) { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); + ListenableFuture executeToStringTransform(JsonNode result) { + if (result.isTextual()) { + return Futures.immediateFuture(result.asText()); } - return result.asBoolean(); + log.warn("Wrong result type: {}", result.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); } @Override public ListenableFuture executeFilterAsync(TbMsg msg) { - ListenableFuture result = executeScriptAsync(msg); - return Futures.transformAsync(result, json -> { - if (!json.isBoolean()) { - log.warn("Wrong result type: {}", json.getNodeType()); - return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); - } else { - return Futures.immediateFuture(json.asBoolean()); - } - }, MoreExecutors.directExecutor()); + return Futures.transformAsync(executeScriptAsync(msg), + this::executeFilterTransform, + MoreExecutors.directExecutor()); + } + + ListenableFuture executeFilterTransform(JsonNode json) { + if (json.isBoolean()) { + return Futures.immediateFuture(json.asBoolean()); + } + log.warn("Wrong result type: {}", json.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); } - ListenableFuture> executeSwitchPostProcessAsyncFunction(JsonNode result) { + ListenableFuture> executeSwitchTransform(JsonNode result) { if (result.isTextual()) { return Futures.immediateFuture(Collections.singleton(result.asText())); } @@ -217,34 +198,19 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S @Override public ListenableFuture> executeSwitchAsync(TbMsg msg) { - log.trace("execute switch async, msg {}", msg); return Futures.transformAsync(executeScriptAsync(msg), - this::executeSwitchPostProcessAsyncFunction, + this::executeSwitchTransform, MoreExecutors.directExecutor()); //usually runs in a callbackExecutor } - private JsonNode executeScript(TbMsg msg) throws ScriptException { - try { - String[] inArgs = prepareArgs(msg); - String eval = sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); - return mapper.readTree(eval); - } catch (ExecutionException e) { - if (e.getCause() instanceof ScriptException) { - throw (ScriptException) e.getCause(); - } else if (e.getCause() instanceof RuntimeException) { - throw new ScriptException(e.getCause().getMessage()); - } else { - throw new ScriptException(e); - } - } catch (Exception e) { - throw new ScriptException(e); - } - } - - private ListenableFuture executeScriptAsync(TbMsg msg) { + ListenableFuture executeScriptAsync(TbMsg msg) { log.trace("execute script async, msg {}", msg); String[] inArgs = prepareArgs(msg); - return Futures.transformAsync(sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]), + return executeScriptAsync(msg.getCustomerId(), inArgs[0], inArgs[1], inArgs[2]); + } + + ListenableFuture executeScriptAsync(CustomerId customerId, Object... args) { + return Futures.transformAsync(sandboxService.invokeFunction(tenantId, customerId, this.scriptId, args), o -> { try { return Futures.immediateFuture(mapper.readTree(o.toString())); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index da9bd93f2b..012e135c60 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -19,14 +19,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.msg.TbMsg; -import javax.script.ScriptException; import java.util.List; import java.util.Set; public interface ScriptEngine { - List executeUpdate(TbMsg msg) throws ScriptException; - ListenableFuture> executeUpdateAsync(TbMsg msg); ListenableFuture executeGenerateAsync(TbMsg prevMsg); @@ -37,11 +34,9 @@ public interface ScriptEngine { ListenableFuture> executeSwitchAsync(TbMsg msg); - JsonNode executeJson(TbMsg msg) throws ScriptException; - - ListenableFuture executeJsonAsync(TbMsg msg) throws ScriptException; + ListenableFuture executeJsonAsync(TbMsg msg); - String executeToString(TbMsg msg) throws ScriptException; + ListenableFuture executeToStringAsync(TbMsg msg); void destroy(); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 43e3a5e329..62ed9b0fa0 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -214,6 +214,10 @@ public interface TbContext { EdgeEventService getEdgeEventService(); + /** + * Js script executors call are completely asynchronous + * */ + @Deprecated ListeningExecutor getJsExecutor(); ListeningExecutor getMailExecutor(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 2b2f0d76a4..c981d5062d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -15,8 +15,11 @@ */ package org.thingsboard.rule.engine.action; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.ListeningExecutor; +import org.checkerframework.checker.nullness.qual.Nullable; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.TbContext; @@ -55,18 +58,21 @@ public class TbLogNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - ListeningExecutor jsExecutor = ctx.getJsExecutor(); ctx.logJsEvalRequest(); - withCallback(jsExecutor.executeAsync(() -> jsEngine.executeToString(msg)), - toString -> { - ctx.logJsEvalResponse(); - log.info(toString); - ctx.tellSuccess(msg); - }, - t -> { - ctx.logJsEvalResponse(); - ctx.tellFailure(msg, t); - }); + Futures.addCallback(jsEngine.executeToStringAsync(msg), new FutureCallback() { + @Override + public void onSuccess(@Nullable String result) { + ctx.logJsEvalResponse(); + log.info(result); + ctx.tellSuccess(msg); + } + + @Override + public void onFailure(Throwable t) { + ctx.logJsEvalResponse(); + ctx.tellFailure(msg, t); + } + }, MoreExecutors.directExecutor()); //usually js responses runs on js callback executor } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java index 5af4ccb29d..bc900a0f2d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java @@ -33,8 +33,6 @@ import org.thingsboard.server.common.msg.TbMsg; import java.util.Set; -import static org.thingsboard.common.util.DonAsynchron.withCallback; - @Slf4j @RuleNode( type = ComponentType.FILTER, diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index fc2e5aa041..4ac81120b0 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -64,7 +64,7 @@ public class TbJsSwitchNodeTest { private RuleNodeId ruleNodeId = new RuleNodeId(Uuids.timeBased()); @Test - public void multipleRoutesAreAllowed() throws TbNodeException, ScriptException { + public void multipleRoutesAreAllowed() throws TbNodeException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("temp", "10"); @@ -72,11 +72,9 @@ public class TbJsSwitchNodeTest { String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); - mockJsExecutor(); when(scriptEngine.executeSwitchAsync(msg)).thenReturn(Futures.immediateFuture(Sets.newHashSet("one", "three"))); node.onMsg(ctx, msg); - verify(ctx).getJsExecutor(); verify(ctx).tellNext(msg, Sets.newHashSet("one", "three")); } @@ -92,19 +90,6 @@ public class TbJsSwitchNodeTest { node.init(ctx, nodeConfiguration); } - @SuppressWarnings("unchecked") - private void mockJsExecutor() { - when(ctx.getJsExecutor()).thenReturn(executor); - doAnswer((Answer>>) invocationOnMock -> { - try { - Callable task = (Callable) (invocationOnMock.getArguments())[0]; - return Futures.immediateFuture((Set) task.call()); - } catch (Throwable th) { - return Futures.immediateFailedFuture(th); - } - }).when(executor).executeAsync(ArgumentMatchers.any(Callable.class)); - } - private void verifyError(TbMsg msg, String message, Class expectedClass) { ArgumentCaptor captor = ArgumentCaptor.forClass(Throwable.class); verify(ctx).tellFailure(same(msg), captor.capture()); From 966b0923e7e293c0bd5fb6aa3f98d43aa9dc93de Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 14:12:33 +0300 Subject: [PATCH 079/101] js-script-engine-api: fixed CE merge --- .../server/service/script/RuleNodeJsScriptEngine.java | 1 + .../main/java/org/thingsboard/rule/engine/api/ScriptEngine.java | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 17e999a533..4ab81702d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -23,6 +23,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index 012e135c60..7ff63319cd 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -28,8 +28,6 @@ public interface ScriptEngine { ListenableFuture executeGenerateAsync(TbMsg prevMsg); - boolean executeFilter(TbMsg msg) throws ScriptException; - ListenableFuture executeFilterAsync(TbMsg msg); ListenableFuture> executeSwitchAsync(TbMsg msg); From 6a63b6e31dd02b9b63178f8bdf77be2cc8740b78 Mon Sep 17 00:00:00 2001 From: Dmitriymush Date: Thu, 17 Jun 2021 14:16:46 +0300 Subject: [PATCH 080/101] fix to entity node level --- .../widget/lib/entities-hierarchy-widget.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts index 5fb11d025b..03d97c50fb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts @@ -443,7 +443,7 @@ export class EntitiesHierarchyWidgetComponent extends PageComponent implements O const dataPageData = subscription.dataPages[0]; const childNodes: HierarchyNavTreeNode[] = []; datasourcesPageData.data.forEach((childDatasource, index) => { - childNodes.push(this.datasourceToNode(childDatasource as HierarchyNodeDatasource, dataPageData.data[index])); + childNodes.push(this.datasourceToNode(childDatasource as HierarchyNodeDatasource, dataPageData.data[index], nodeCtx)); }); nodeCtx.childrenNodesLoaded = true; childrenNodesLoadCb(this.prepareNodes(childNodes)); From 29910fad8f6634e4fa7cfc1db2cf74d2e723f7d8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:34:54 +0300 Subject: [PATCH 081/101] TbStopWatch updated licence header for CE --- .../thingsboard/common/util/TbStopWatch.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index b7dbd0052c..155ffb5360 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.common.util; From 75801045cfcdda85b90973d7e3079b22b747de52 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 17 Jun 2021 15:08:00 +0300 Subject: [PATCH 082/101] UI: Add SNMP device profile configuration --- .../home/components/home-components.module.ts | 5 +- ...ofile-transport-configuration.component.ts | 4 +- ...ile-transport-configuration.component.html | 23 -- ...rofile-communication-config.component.html | 72 ++++++ ...rofile-communication-config.component.scss | 44 ++++ ...-profile-communication-config.component.ts | 218 ++++++++++++++++++ ...snmp-device-profile-mapping.component.html | 81 +++++++ ...snmp-device-profile-mapping.component.scss | 46 ++++ .../snmp-device-profile-mapping.component.ts | 165 +++++++++++++ ...ile-transport-configuration.component.html | 46 ++++ ...ofile-transport-configuration.component.ts | 57 +++-- .../snmp-device-profile-transport.module.ts | 38 +++ ui-ngx/src/app/shared/models/constants.ts | 16 ++ ui-ngx/src/app/shared/models/device.models.ts | 37 ++- .../assets/locale/locale.constant-en_US.json | 25 ++ 15 files changed, 834 insertions(+), 43 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html rename ui-ngx/src/app/modules/home/components/profile/device/{ => snpm}/snmp-device-profile-transport-configuration.component.ts (66%) create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 3958830088..d0ad42b62e 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -99,7 +99,6 @@ import { DeviceProfileDialogComponent } from '@home/components/profile/device-pr import { DeviceProfileAutocompleteComponent } from '@home/components/profile/device-profile-autocomplete.component'; import { MqttDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/mqtt-device-profile-transport-configuration.component'; import { CoapDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/coap-device-profile-transport-configuration.component'; -import { SnmpDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/snmp-device-profile-transport-configuration.component'; import { DeviceProfileAlarmsComponent } from '@home/components/profile/alarm/device-profile-alarms.component'; import { DeviceProfileAlarmComponent } from '@home/components/profile/alarm/device-profile-alarm.component'; import { CreateAlarmRulesComponent } from '@home/components/profile/alarm/create-alarm-rules.component'; @@ -143,6 +142,7 @@ import { SecurityConfigLwm2mComponent } from '@home/components/device/security-c import { SecurityConfigLwm2mServerComponent } from '@home/components/device/security-config-lwm2m-server.component'; import { DashboardImageDialogComponent } from '@home/components/dashboard-page/dashboard-image-dialog.component'; import { WidgetContainerComponent } from '@home/components/widget/widget-container.component'; +import { SnmpDeviceProfileTransportModule } from '@home/components/profile/device/snpm/snmp-device-profile-transport.module'; @NgModule({ declarations: @@ -228,7 +228,6 @@ import { WidgetContainerComponent } from '@home/components/widget/widget-contain DefaultDeviceProfileTransportConfigurationComponent, MqttDeviceProfileTransportConfigurationComponent, CoapDeviceProfileTransportConfigurationComponent, - SnmpDeviceProfileTransportConfigurationComponent, DeviceProfileTransportConfigurationComponent, CreateAlarmRulesComponent, AlarmRuleComponent, @@ -272,6 +271,7 @@ import { WidgetContainerComponent } from '@home/components/widget/widget-contain SharedModule, SharedHomeComponentsModule, Lwm2mProfileComponentsModule, + SnmpDeviceProfileTransportModule, StatesControllerModule ], exports: [ @@ -339,7 +339,6 @@ import { WidgetContainerComponent } from '@home/components/widget/widget-contain DefaultDeviceProfileTransportConfigurationComponent, MqttDeviceProfileTransportConfigurationComponent, CoapDeviceProfileTransportConfigurationComponent, - SnmpDeviceProfileTransportConfigurationComponent, DeviceProfileTransportConfigurationComponent, CreateAlarmRulesComponent, AlarmRuleComponent, diff --git a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts index 305a0a84b6..50b3bc801c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts @@ -89,7 +89,9 @@ export class DeviceProfileTransportConfigurationComponent implements ControlValu if (configuration) { delete configuration.type; } - this.deviceProfileTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + setTimeout(() => { + this.deviceProfileTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + }); } private updateModel() { diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html deleted file mode 100644 index f57b1ac716..0000000000 --- a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html +++ /dev/null @@ -1,23 +0,0 @@ - -
- - -
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html new file mode 100644 index 0000000000..7d2f8a154c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html @@ -0,0 +1,72 @@ + +
+
+
+ + device-profile.snmp.scope + + + {{ snmpSpecTypeTranslationMap.get(snmpSpecType) }} + + + + {{ 'device-profile.snmp.scope-required' | translate }} + + + +
+ + device-profile.snmp.querying-frequency + + + {{ 'device-profile.snmp.querying-frequency-required' | translate }} + + + {{ 'device-profile.snmp.querying-frequency-invalid-format' | translate }} + + + + +
+
+ +
+
+ device-profile.snmp.please-add-communication-config +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss new file mode 100644 index 0000000000..3b6661b862 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .communication-config { + border: 2px groove rgba(0, 0, 0, 0.25); + border-radius: 4px; + padding: 8px; + min-width: 0; + } + + .scope-row { + padding-bottom: 8px; + } + + .required-text { + margin: 16px 0 + } +} + +:host ::ng-deep { + .mat-form-field.spec { + .mat-form-field-infix { + width: 160px; + } + } + .button-icon{ + font-size: 20px; + width: 20px; + height: 20px; + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts new file mode 100644 index 0000000000..d683d46880 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts @@ -0,0 +1,218 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { SnmpCommunicationConfig, SnmpSpecType, SnmpSpecTypeTranslationMap } from '@shared/models/device.models'; +import { Subject, Subscription } from 'rxjs'; +import { isUndefinedOrNull } from '@core/utils'; +import { takeUntil } from 'rxjs/operators'; + +@Component({ + selector: 'tb-snmp-device-profile-communication-config', + templateUrl: './snmp-device-profile-communication-config.component.html', + styleUrls: ['./snmp-device-profile-communication-config.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceProfileCommunicationConfigComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceProfileCommunicationConfigComponent), + multi: true + }] +}) +export class SnmpDeviceProfileCommunicationConfigComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { + + snmpSpecTypes = Object.values(SnmpSpecType); + snmpSpecTypeTranslationMap = SnmpSpecTypeTranslationMap; + + deviceProfileCommunicationConfig: FormGroup; + + @Input() + disabled: boolean; + + private usedSpecType: SnmpSpecType[] = []; + private valueChange$: Subscription = null; + private destroy$ = new Subject(); + private propagateChange = (v: any) => { }; + + constructor(private fb: FormBuilder) { } + + ngOnInit(): void { + this.deviceProfileCommunicationConfig = this.fb.group({ + communicationConfig: this.fb.array([]) + }); + } + + ngOnDestroy() { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + this.destroy$.next(); + this.destroy$.complete(); + } + + communicationConfigFormArray(): FormArray { + return this.deviceProfileCommunicationConfig.get('communicationConfig') as FormArray; + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean) { + this.disabled = isDisabled; + if (this.disabled) { + this.deviceProfileCommunicationConfig.disable({emitEvent: false}); + } else { + this.deviceProfileCommunicationConfig.enable({emitEvent: false}); + } + } + + writeValue(communicationConfig: SnmpCommunicationConfig[]) { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + const communicationConfigControl: Array = []; + if (communicationConfig) { + communicationConfig.forEach((config) => { + communicationConfigControl.push(this.createdFormGroup(config)); + }); + } + this.deviceProfileCommunicationConfig.setControl('communicationConfig', this.fb.array(communicationConfigControl)); + if (!communicationConfig || !communicationConfig.length) { + this.addCommunicationConfig(); + } + if (this.disabled) { + this.deviceProfileCommunicationConfig.disable({emitEvent: false}); + } else { + this.deviceProfileCommunicationConfig.enable({emitEvent: false}); + } + this.valueChange$ = this.deviceProfileCommunicationConfig.valueChanges.subscribe(() => { + this.updateModel(); + }); + this.updateUsedSpecType(); + if (!this.disabled && !this.deviceProfileCommunicationConfig.valid) { + this.updateModel(); + } + } + + public validate() { + return this.deviceProfileCommunicationConfig.valid && this.deviceProfileCommunicationConfig.value.communicationConfig.length ? null : { + communicationConfig: false + }; + } + + public removeCommunicationConfig(index: number) { + this.communicationConfigFormArray().removeAt(index); + } + + + get isAddEnabled(): boolean { + return this.communicationConfigFormArray().length !== Object.keys(SnmpSpecType).length; + } + + public addCommunicationConfig() { + this.communicationConfigFormArray().push(this.createdFormGroup()); + this.deviceProfileCommunicationConfig.updateValueAndValidity(); + if (!this.deviceProfileCommunicationConfig.valid) { + this.updateModel(); + } + } + + private getFirstUnusedSeverity(): SnmpSpecType { + for (const type of Object.values(SnmpSpecType)) { + if (this.usedSpecType.indexOf(type) === -1) { + return type; + } + } + return null; + } + + public isDisabledSeverity(type: SnmpSpecType, index: number): boolean { + const usedIndex = this.usedSpecType.indexOf(type); + return usedIndex > -1 && usedIndex !== index; + } + + public isShowFrequency(type: SnmpSpecType): boolean { + return type === SnmpSpecType.TELEMETRY_QUERYING || type === SnmpSpecType.CLIENT_ATTRIBUTES_QUERYING; + } + + private updateUsedSpecType() { + this.usedSpecType = []; + const value: SnmpCommunicationConfig[] = this.deviceProfileCommunicationConfig.get('communicationConfig').value; + value.forEach((rule, index) => { + this.usedSpecType[index] = rule.spec; + }); + } + + private createdFormGroup(value?: SnmpCommunicationConfig): FormGroup { + if (isUndefinedOrNull(value)) { + value = { + spec: this.getFirstUnusedSeverity(), + queryingFrequencyMs: 0, + mappings: null + }; + } + const form = this.fb.group({ + spec: [value.spec, Validators.required], + mappings: [value.mappings] + }); + if (this.isShowFrequency(value.spec)) { + form.addControl('queryingFrequencyMs', + this.fb.control(value.queryingFrequencyMs, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); + } + form.get('spec').valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(spec => { + if (this.isShowFrequency(spec)) { + form.addControl('queryingFrequencyMs', + this.fb.control(0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); + } else { + form.removeControl('queryingFrequencyMs'); + } + }); + return form; + } + + private updateModel() { + const value: SnmpCommunicationConfig[] = this.deviceProfileCommunicationConfig.get('communicationConfig').value; + value.forEach(config => { + if (!this.isShowFrequency(config.spec)) { + delete config.queryingFrequencyMs; + } + }); + this.updateUsedSpecType(); + this.propagateChange(value); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html new file mode 100644 index 0000000000..a2a0a97d15 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html @@ -0,0 +1,81 @@ + +
+
+
+ + + + +
+
+ +
+
+ + + + + {{ dataTypesTranslationMap.get(dataType) | translate }} + + + + {{ 'device-profile.snmp.data-type-required' | translate }} + + + + + + + {{ 'device-profile.snmp.data-key-required' | translate }} + + + + + + + {{ 'device-profile.snmp.oid-required' | translate }} + + + {{ 'device-profile.snmp.oid-pattern' | translate }} + + + +
+
+
+ device-profile.snmp.please-add-mapping-config +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss new file mode 100644 index 0000000000..94cf2290ac --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .mapping-config { + min-width: 518px; + } + .mapping-list { + padding-bottom: 8px; + height: 46px; + } + + .required-text { + margin: 14px 0; + } +} + +:host ::ng-deep { + .mapping-list { + mat-form-field { + .mat-form-field-wrapper { + padding-bottom: 0; + .mat-form-field-infix { + border-top-width: 0.2em; + width: auto; + min-width: auto; + } + .mat-form-field-underline { + bottom: 0; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts new file mode 100644 index 0000000000..53dbec422c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts @@ -0,0 +1,165 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { SnmpMapping } from '@shared/models/device.models'; +import { Subscription } from 'rxjs'; +import { DataType, DataTypeTranslationMap } from '@shared/models/constants'; +import { isUndefinedOrNull } from '@core/utils'; + +@Component({ + selector: 'tb-snmp-device-profile-mapping', + templateUrl: './snmp-device-profile-mapping.component.html', + styleUrls: ['./snmp-device-profile-mapping.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceProfileMappingComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceProfileMappingComponent), + multi: true + }] +}) +export class SnmpDeviceProfileMappingComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { + + mappingsConfigForm: FormGroup; + + dataTypes = Object.values(DataType); + dataTypesTranslationMap = DataTypeTranslationMap; + + @Input() + disabled: boolean; + + private readonly oidPattern: RegExp = /^\.?([0-2])((\.0)|(\.[1-9][0-9]*))*$/; + + private valueChange$: Subscription = null; + private propagateChange = (v: any) => { }; + + constructor(private fb: FormBuilder) { } + + ngOnInit() { + this.mappingsConfigForm = this.fb.group({ + mappings: this.fb.array([]) + }); + } + + ngOnDestroy() { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(fn: any) { + } + + setDisabledState(isDisabled: boolean) { + this.disabled = isDisabled; + if (this.disabled) { + this.mappingsConfigForm.disable({emitEvent: false}); + } else { + this.mappingsConfigForm.enable({emitEvent: false}); + } + } + + validate(): ValidationErrors | null { + return this.mappingsConfigForm.valid && this.mappingsConfigForm.value.mappings.length ? null : { + mapping: false + }; + } + + writeValue(mappings: SnmpMapping[]) { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + const mappingsControl: Array = []; + if (mappings) { + mappings.forEach((config) => { + mappingsControl.push(this.createdFormGroup(config)); + }); + } + this.mappingsConfigForm.setControl('mappings', this.fb.array(mappingsControl)); + if (!mappings || !mappings.length) { + this.addMappingConfig(); + } + if (this.disabled) { + this.mappingsConfigForm.disable({emitEvent: false}); + } else { + this.mappingsConfigForm.enable({emitEvent: false}); + } + this.valueChange$ = this.mappingsConfigForm.valueChanges.subscribe(() => { + this.updateModel(); + }); + if (!this.disabled && !this.mappingsConfigForm.valid) { + this.updateModel(); + } + } + + mappingsConfigFormArray(): FormArray { + return this.mappingsConfigForm.get('mappings') as FormArray; + } + + public addMappingConfig() { + this.mappingsConfigFormArray().push(this.createdFormGroup()); + this.mappingsConfigForm.updateValueAndValidity(); + if (!this.mappingsConfigForm.valid) { + this.updateModel(); + } + } + + public removeMappingConfig(index: number) { + this.mappingsConfigFormArray().removeAt(index); + } + + private createdFormGroup(value?: SnmpMapping): FormGroup { + if (isUndefinedOrNull(value)) { + value = { + dataType: DataType.STRING, + key: '', + oid: '' + }; + } + return this.fb.group({ + dataType: [value.dataType, Validators.required], + key: [value.key, Validators.required], + oid: [value.oid, [Validators.required, Validators.pattern(this.oidPattern)]] + }); + } + + private updateModel() { + const value: SnmpMapping[] = this.mappingsConfigForm.get('mappings').value; + this.propagateChange(value); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html new file mode 100644 index 0000000000..7dad337d72 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html @@ -0,0 +1,46 @@ + +
+
+ + device-profile.snmp.timeout-ms + + + {{ 'device-profile.snmp.timeout-ms-required' | translate }} + + + {{ 'device-profile.snmp.timeout-ms-invalid-format' | translate }} + + + + device-profile.snmp.retries + + + {{ 'device-profile.snmp.retries-required' | translate }} + + + {{ 'device-profile.snmp.retries-invalid-format' | translate }} + + +
+
device-profile.snmp.communication-configs
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts similarity index 66% rename from ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts rename to ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts index e6749a9219..a384acbae3 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts @@ -15,9 +15,16 @@ /// import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; -import { Store } from '@ngrx/store'; -import { AppState } from '@app/core/core.state'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { DeviceProfileTransportConfiguration, @@ -40,19 +47,24 @@ export interface OidMappingConfiguration { selector: 'tb-snmp-device-profile-transport-configuration', templateUrl: './snmp-device-profile-transport-configuration.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => SnmpDeviceProfileTransportConfigurationComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceProfileTransportConfigurationComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceProfileTransportConfigurationComponent), + multi: true + }] }) -export class SnmpDeviceProfileTransportConfigurationComponent implements ControlValueAccessor, OnInit, OnDestroy { +export class SnmpDeviceProfileTransportConfigurationComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { snmpDeviceProfileTransportConfigurationFormGroup: FormGroup; private destroy$ = new Subject(); private requiredValue: boolean; - private configuration = []; get required(): boolean { return this.requiredValue; @@ -69,12 +81,14 @@ export class SnmpDeviceProfileTransportConfigurationComponent implements Control private propagateChange = (v: any) => { } - constructor(private store: Store, private fb: FormBuilder) { + constructor(private fb: FormBuilder) { } ngOnInit(): void { this.snmpDeviceProfileTransportConfigurationFormGroup = this.fb.group({ - configuration: [null, Validators.required] + timeoutMs: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + retries: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + communicationConfigs: [null, Validators.required], }); this.snmpDeviceProfileTransportConfigurationFormGroup.valueChanges.pipe( takeUntil(this.destroy$) @@ -95,18 +109,33 @@ export class SnmpDeviceProfileTransportConfigurationComponent implements Control registerOnTouched(fn: any): void { } + setDisabledState(isDisabled: boolean) { + this.disabled = isDisabled; + if (this.disabled) { + this.snmpDeviceProfileTransportConfigurationFormGroup.disable({emitEvent: false}); + } else { + this.snmpDeviceProfileTransportConfigurationFormGroup.enable({emitEvent: false}); + } + } + writeValue(value: SnmpDeviceProfileTransportConfiguration | null): void { if (isDefinedAndNotNull(value)) { - this.snmpDeviceProfileTransportConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + this.snmpDeviceProfileTransportConfigurationFormGroup.patchValue(value, {emitEvent: !value.communicationConfigs}); } } private updateModel() { let configuration: DeviceProfileTransportConfiguration = null; if (this.snmpDeviceProfileTransportConfigurationFormGroup.valid) { - configuration = this.snmpDeviceProfileTransportConfigurationFormGroup.getRawValue().configuration; + configuration = this.snmpDeviceProfileTransportConfigurationFormGroup.getRawValue(); configuration.type = DeviceTransportType.SNMP; } this.propagateChange(configuration); } + + validate(): ValidationErrors | null { + return this.snmpDeviceProfileTransportConfigurationFormGroup.valid ? null : { + snmpDeviceProfileTransportConfiguration: false + }; + } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts new file mode 100644 index 0000000000..c0797d9b0e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts @@ -0,0 +1,38 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { SharedModule } from '@shared/shared.module'; +import { CommonModule } from '@angular/common'; +import { SnmpDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component'; +import { SnmpDeviceProfileCommunicationConfigComponent } from './snmp-device-profile-communication-config.component'; +import { SnmpDeviceProfileMappingComponent } from './snmp-device-profile-mapping.component'; + +@NgModule({ + declarations: [ + SnmpDeviceProfileTransportConfigurationComponent, + SnmpDeviceProfileCommunicationConfigComponent, + SnmpDeviceProfileMappingComponent + ], + imports: [ + CommonModule, + SharedModule + ], + exports: [ + SnmpDeviceProfileTransportConfigurationComponent + ] +}) +export class SnmpDeviceProfileTransportModule { } diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index b72e2b196d..cd81753c76 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -147,6 +147,22 @@ export enum ValueType { JSON = 'JSON' } +export enum DataType { + STRING = 'STRING', + LONG = 'LONG', + BOOLEAN = 'BOOLEAN', + DOUBLE = 'DOUBLE', + JSON = 'JSON' +} + +export const DataTypeTranslationMap = new Map([ + [DataType.STRING, 'value.string'], + [DataType.LONG, 'value.integer'], + [DataType.BOOLEAN, 'value.boolean'], + [DataType.DOUBLE, 'value.double'], + [DataType.JSON, 'value.json'] +]); + export const valueTypesMap = new Map( [ [ diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 7af8e56472..568faeb452 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -29,6 +29,7 @@ import * as _moment from 'moment'; import { AbstractControl, ValidationErrors } from '@angular/forms'; import { OtaPackageId } from '@shared/models/id/ota-package-id'; import { DashboardId } from '@shared/models/id/dashboard-id'; +import { DataType } from '@shared/models/constants'; export enum DeviceProfileType { DEFAULT = 'DEFAULT', @@ -257,7 +258,35 @@ export interface Lwm2mDeviceProfileTransportConfiguration { } export interface SnmpDeviceProfileTransportConfiguration { - [key: string]: any; + timeoutMs?: number; + retries?: number; + communicationConfigs?: SnmpCommunicationConfig[]; +} + +export enum SnmpSpecType { + TELEMETRY_QUERYING = 'TELEMETRY_QUERYING', + CLIENT_ATTRIBUTES_QUERYING = 'CLIENT_ATTRIBUTES_QUERYING', + SHARED_ATTRIBUTES_SETTING = 'SHARED_ATTRIBUTES_SETTING', + TO_DEVICE_RPC_REQUEST = 'TO_DEVICE_RPC_REQUEST' +} + +export const SnmpSpecTypeTranslationMap = new Map([ + [SnmpSpecType.TELEMETRY_QUERYING, ' Telemetry'], + [SnmpSpecType.CLIENT_ATTRIBUTES_QUERYING, 'Client attributes'], + [SnmpSpecType.SHARED_ATTRIBUTES_SETTING, 'Shared attributes'], + [SnmpSpecType.TO_DEVICE_RPC_REQUEST, 'RPC request'] +]); + +export interface SnmpCommunicationConfig { + spec: SnmpSpecType; + mappings: SnmpMapping[]; + queryingFrequencyMs?: number; +} + +export interface SnmpMapping { + oid: string; + key: string; + dataType: DataType; } export type DeviceProfileTransportConfigurations = DefaultDeviceProfileTransportConfiguration & @@ -332,7 +361,11 @@ export function createDeviceProfileTransportConfiguration(type: DeviceTransportT transportConfiguration = {...lwm2mTransportConfiguration, type: DeviceTransportType.LWM2M}; break; case DeviceTransportType.SNMP: - const snmpTransportConfiguration: SnmpDeviceProfileTransportConfiguration = {}; + const snmpTransportConfiguration: SnmpDeviceProfileTransportConfiguration = { + timeoutMs: 0, + retries: 0, + communicationConfigs: null + }; transportConfiguration = {...snmpTransportConfiguration, type: DeviceTransportType.SNMP}; break; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ec084b8a3a..95dc0c2d72 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1297,6 +1297,31 @@ "sw-update-recourse": "Software update CoAP recourse", "sw-update-recourse-required": "Software update CoAP recourse is required.", "config-json-tab": "Json Config Profile Device" + }, + "snmp": { + "add-communication-config": "Add communication config", + "add-mapping": "Add mapping", + "communication-configs": "Communication configs", + "data-key": "Data key", + "data-key-required": "Data key is required.", + "data-type": "Data type", + "data-type-required": "Data type is required.", + "oid": "OID", + "oid-pattern": "Invalid OID format.", + "oid-required": "OID is required.", + "please-add-communication-config": "Please add communication config", + "please-add-mapping-config": "Please add mapping config", + "querying-frequency": "Querying frequency, ms", + "querying-frequency-invalid-format": "Querying frequency must be a positive integer.", + "querying-frequency-required": "Querying frequency is required.", + "retries": "Retries", + "retries-invalid-format": "Retries must be a positive integer.", + "retries-required": "Retries is required.", + "scope": "Scope", + "scope-required": "Scope is required.", + "timeout-ms": "Timeout, ms", + "timeout-ms-invalid-format": "Timeout must be a positive integer.", + "timeout-ms-required": "Timeout is required." } }, "dialog": { From e4dd517edd1a0d471a4ea0aa3e687ef26184713e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 11 Jun 2021 12:36:03 +0300 Subject: [PATCH 083/101] cache: added cache executor service --- .../src/main/resources/thingsboard.yml | 1 + .../dao/cache/CacheExecutorService.java | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 54721ba2fa..ffdb783b12 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -332,6 +332,7 @@ actors: cache: # caffeine or redis type: "${CACHE_TYPE:caffeine}" + maximumPoolSize: "${CACHE_MAXIMUM_POOL_SIZE:16}" # max pool size to process futures that calls the external cache attributes: # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java new file mode 100644 index 0000000000..ca00e41d79 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java @@ -0,0 +1,48 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao.cache; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.AbstractListeningExecutor; + +@Component +public class CacheExecutorService extends AbstractListeningExecutor { + + @Value("${cache.maximumPoolSize}") + private int poolSize; + + @Override + protected int getThreadPollSize() { + return poolSize; + } + +} From c843b2f09e67a084e11a34d8c7f7f81075adc4b8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 11 Jun 2021 12:36:51 +0300 Subject: [PATCH 084/101] cache: CachedAttributesService replaced directExecutor with CacheExecutorService --- .../dao/attributes/CachedAttributesService.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 87ddfeee9a..92c75760e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -17,11 +17,9 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; @@ -32,6 +30,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.cache.CacheExecutorService; import org.thingsboard.server.dao.service.Validator; import java.util.ArrayList; @@ -45,7 +44,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @Service @@ -59,12 +57,15 @@ public class CachedAttributesService implements AttributesService { private final AttributesCacheWrapper cacheWrapper; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; + private final CacheExecutorService cacheExecutorService; public CachedAttributesService(AttributesDao attributesDao, AttributesCacheWrapper cacheWrapper, - StatsFactory statsFactory) { + StatsFactory statsFactory, + CacheExecutorService cacheExecutorService) { this.attributesDao = attributesDao; this.cacheWrapper = cacheWrapper; + this.cacheExecutorService = cacheExecutorService; this.hitCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "hit"); this.missCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "miss"); @@ -88,7 +89,7 @@ public class CachedAttributesService implements AttributesService { // TODO: think if it's a good idea to store 'empty' attributes cacheWrapper.put(attributeCacheKey, foundAttrKvEntry.orElse(null)); return foundAttrKvEntry; - }, MoreExecutors.directExecutor()); + }, cacheExecutorService); } } @@ -111,7 +112,7 @@ public class CachedAttributesService implements AttributesService { notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); ListenableFuture> result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); - return Futures.transform(result, foundInDbAttributes -> mergeDbAndCacheAttributes(entityId, scope, cachedAttributes, notFoundAttributeKeys, foundInDbAttributes), MoreExecutors.directExecutor()); + return Futures.transform(result, foundInDbAttributes -> mergeDbAndCacheAttributes(entityId, scope, cachedAttributes, notFoundAttributeKeys, foundInDbAttributes), cacheExecutorService); } @@ -169,7 +170,7 @@ public class CachedAttributesService implements AttributesService { // TODO: can do if (attributesCache.get() != null) attributesCache.put() instead, but will be more twice more requests to cache List attributeKeys = attributes.stream().map(KvEntry::getKey).collect(Collectors.toList()); - future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor()); + future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), cacheExecutorService); return future; } @@ -177,7 +178,7 @@ public class CachedAttributesService implements AttributesService { public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { validate(entityId, scope); ListenableFuture> future = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); - future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor()); + future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), cacheExecutorService); return future; } From be61b3c841328b7d0d4d5570512fefcd9d852b30 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 14 Jun 2021 14:39:00 +0300 Subject: [PATCH 085/101] cache: CachedAttributesService Executor choice depends on local or remote cache type. For local - directExecutor, for remote (IO calls) - dedicated threadPool to unblock any caller thread. Test added. --- .../attributes/CachedAttributesService.java | 1 + .../CachedAttributesServiceTest.java | 88 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 92c75760e2..d4e29f02de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.Cache; diff --git a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java new file mode 100644 index 0000000000..af980a6301 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java @@ -0,0 +1,88 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao.attributes; + +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.Test; +import org.thingsboard.server.dao.cache.CacheConfiguration; +import org.thingsboard.server.dao.cache.CacheExecutorService; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.Mockito.mock; + +public class CachedAttributesServiceTest { + + public static final String REDIS = "redis"; + + @Test + public void givenLocalCacheTypeName_whenEquals_thenOK() { + assertThat(CachedAttributesService.LOCAL_CACHE_TYPE, is("caffeine")); + } + + @Test + public void givenCacheType_whenGetExecutor_thenDirectExecutor() { + CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); + CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); + willCallRealMethod().given(cachedAttributesService).getExecutor(any(), any()); + + assertThat(cachedAttributesService.getExecutor(null, cacheExecutorService), is(MoreExecutors.directExecutor())); + + CacheConfiguration cacheConfiguration = new CacheConfiguration(); + cacheConfiguration.setType(null); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); + + cacheConfiguration.setType(""); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); + + cacheConfiguration.setType(CachedAttributesService.LOCAL_CACHE_TYPE); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); + + } + + @Test + public void givenCacheType_whenGetExecutor_thenReturnCacheExecutorService() { + CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); + CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); + willCallRealMethod().given(cachedAttributesService).getExecutor(any(CacheConfiguration.class), any(CacheExecutorService.class)); + + CacheConfiguration cacheConfiguration = new CacheConfiguration(); + cacheConfiguration.setType(REDIS); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); + + cacheConfiguration.setType("unknownCacheType"); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); + + } + +} \ No newline at end of file From 0c1e608051ac4e77cade38a27587635351e54737 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 17 Jun 2021 16:35:30 +0300 Subject: [PATCH 086/101] UI: Refactoring lwm2m object list --- .../device/lwm2m/lwm2m-object-list.component.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts index ae7c3ccdc6..62d96b69e0 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts @@ -89,6 +89,13 @@ export class Lwm2mObjectListComponent implements ControlValueAccessor, OnInit, V objectsList: [this.objectsList], objectLwm2m: [''] }); + this.lwm2mListFormGroup.valueChanges.subscribe((value) => { + let formValue = null; + if (this.lwm2mListFormGroup.valid) { + formValue = value; + } + this.propagateChange(formValue); + }); } private updateValidators = (): void => { @@ -142,7 +149,7 @@ export class Lwm2mObjectListComponent implements ControlValueAccessor, OnInit, V this.objectsList = []; this.modelValue = []; } - this.lwm2mListFormGroup.get('objectsList').setValue(this.objectsList, {emitEvents: false}); + this.lwm2mListFormGroup.patchValue({objectsList: this.objectsList}, {emitEvent: false}); this.dirty = false; } } @@ -195,9 +202,9 @@ export class Lwm2mObjectListComponent implements ControlValueAccessor, OnInit, V } } - private clear = (): void => { + private clear() { this.searchText = ''; - this.lwm2mListFormGroup.get('objectLwm2m').patchValue(null); + this.lwm2mListFormGroup.get('objectLwm2m').patchValue(null, {emitEvent: false}); setTimeout(() => { this.objectInput.nativeElement.blur(); this.objectInput.nativeElement.focus(); From 248e35cb249eff03b25e8b983701181191e0ada1 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 17 Jun 2021 17:05:55 +0300 Subject: [PATCH 087/101] UI: Fixed not updated view entities table, after updated values --- .../home/components/widget/lib/alarms-table-widget.component.ts | 1 + .../components/widget/lib/entities-table-widget.component.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index 138c56d307..390559bab3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -274,6 +274,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.updateTitle(true); this.alarmsDatasource.updateAlarms(); this.clearCache(); + this.ctx.detectChanges(); } public pageLinkSortDirection(): SortDirection { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index 8994051223..68db8a0dcc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -47,7 +47,6 @@ import { isDefined, isNumber, isObject, - isString, isUndefined } from '@core/utils'; import cssjs from '@core/css/css'; @@ -236,6 +235,7 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni this.updateTitle(true); this.entityDatasource.dataUpdated(); this.clearCache(); + this.ctx.detectChanges(); } public pageLinkSortDirection(): SortDirection { From bed454ec0fcace61736ca58129d1b74646fdffba Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Jun 2021 18:29:33 +0300 Subject: [PATCH 088/101] LwM2M Refactoring of OTA --- .../src/main/resources/thingsboard.yml | 7 +- .../transport/TransportSqlTestSuite.java | 14 +- .../config/LwM2MTransportServerConfig.java | 20 +- .../server/DefaultLwM2mTransportService.java | 5 +- .../lwm2m/server/LwM2mNetworkConfig.java | 6 +- .../server/LwM2mTransportCoapResource.java | 9 +- .../lwm2m/server/LwM2mTransportUtil.java | 46 +- .../lwm2m/server/UpdateResultFw.java | 13 +- .../DefaultLwM2MAttributesService.java | 14 +- .../lwm2m/server/client/LwM2mClient.java | 22 - .../lwm2m/server/client/LwM2mFwSwUpdate.java | 486 ------------------ .../common/LwM2MExecutorAwareService.java | 41 ++ .../AbstractTbLwM2MRequestCallback.java | 12 +- .../DefaultLwM2mDownlinkMsgHandler.java | 62 ++- .../TbLwM2MCancelAllObserveCallback.java | 8 +- .../TbLwM2MCancelObserveCallback.java | 7 +- .../downlink/TbLwM2MDeleteCallback.java | 5 +- .../downlink/TbLwM2MDiscoverCallback.java | 5 +- .../downlink/TbLwM2MExecuteCallback.java | 5 +- .../downlink/TbLwM2MObserveCallback.java | 7 +- .../server/downlink/TbLwM2MReadCallback.java | 7 +- .../downlink/TbLwM2MTargetedCallback.java | 7 +- .../TbLwM2MUplinkTargetedCallback.java | 35 ++ .../TbLwM2MWriteAttributesCallback.java | 5 +- .../TbLwM2MWriteResponseCallback.java | 7 +- .../log/DefaultLwM2MTelemetryLogService.java | 56 ++ .../server/log/LwM2MTelemetryLogService.java | 26 + .../ota/DefaultLwM2MOtaUpdateService.java | 163 ++++-- .../lwm2m/server/ota/LwM2MClientOtaInfo.java | 23 +- .../server/ota/LwM2MOtaUpdateService.java | 6 +- .../rpc/DefaultLwM2MRpcRequestHandler.java | 22 +- .../uplink/DefaultLwM2MUplinkMsgHandler.java | 197 ++----- .../server/uplink/LwM2mUplinkMsgHandler.java | 4 - .../src/main/resources/tb-lwm2m-transport.yml | 8 +- 34 files changed, 479 insertions(+), 881 deletions(-) delete mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/common/LwM2MExecutorAwareService.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MUplinkTargetedCallback.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/DefaultLwM2MTelemetryLogService.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/LwM2MTelemetryLogService.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 54721ba2fa..fcde169fc9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -677,12 +677,11 @@ transport: timeout: "${LWM2M_TIMEOUT:120000}" recommended_ciphers: "${LWM2M_RECOMMENDED_CIPHERS:false}" recommended_supported_groups: "${LWM2M_RECOMMENDED_SUPPORTED_GROUPS:true}" - response_pool_size: "${LWM2M_RESPONSE_POOL_SIZE:100}" - registered_pool_size: "${LWM2M_REGISTERED_POOL_SIZE:10}" + uplink_pool_size: "${LWM2M_UPLINK_POOL_SIZE:10}" + downlink_pool_size: "${LWM2M_DOWNLINK_POOL_SIZE:10}" + ota_pool_size: "${LWM2M_OTA_POOL_SIZE:10}" registration_store_pool_size: "${LWM2M_REGISTRATION_STORE_POOL_SIZE:100}" clean_period_in_sec: "${LWM2M_CLEAN_PERIOD_IN_SEC:2}" - update_registered_pool_size: "${LWM2M_UPDATE_REGISTERED_POOL_SIZE:10}" - un_registered_pool_size: "${LWM2M_UN_REGISTERED_POOL_SIZE:10}" log_max_length: "${LWM2M_LOG_MAX_LENGTH:100}" # Use redis for Security and Registration stores redis.enabled: "${LWM2M_REDIS_ENABLED:false}" diff --git a/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java index d059ea1449..25df3bee00 100644 --- a/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java @@ -26,13 +26,13 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({ -// "org.thingsboard.server.transport.*.rpc.sql.*Test", -// "org.thingsboard.server.transport.*.telemetry.timeseries.sql.*Test", -// "org.thingsboard.server.transport.*.telemetry.attributes.sql.*Test", -// "org.thingsboard.server.transport.*.attributes.updates.sql.*Test", -// "org.thingsboard.server.transport.*.attributes.request.sql.*Test", -// "org.thingsboard.server.transport.*.claim.sql.*Test", -// "org.thingsboard.server.transport.*.provision.sql.*Test", + "org.thingsboard.server.transport.*.rpc.sql.*Test", + "org.thingsboard.server.transport.*.telemetry.timeseries.sql.*Test", + "org.thingsboard.server.transport.*.telemetry.attributes.sql.*Test", + "org.thingsboard.server.transport.*.attributes.updates.sql.*Test", + "org.thingsboard.server.transport.*.attributes.request.sql.*Test", + "org.thingsboard.server.transport.*.claim.sql.*Test", + "org.thingsboard.server.transport.*.provision.sql.*Test", "org.thingsboard.server.transport.lwm2m.*Test" }) public class TransportSqlTestSuite { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java index 25c7766895..a2ff361712 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java @@ -57,29 +57,21 @@ public class LwM2MTransportServerConfig implements LwM2MSecureServerConfig { private boolean recommendedSupportedGroups; @Getter - @Value("${transport.lwm2m.response_pool_size:}") - private int responsePoolSize; + @Value("${transport.lwm2m.downlink_pool_size:}") + private int downlinkPoolSize; @Getter - @Value("${transport.lwm2m.registered_pool_size:}") - private int registeredPoolSize; + @Value("${transport.lwm2m.uplink_pool_size:}") + private int uplinkPoolSize; @Getter - @Value("${transport.lwm2m.registration_store_pool_size:}") - private int registrationStorePoolSize; + @Value("${transport.lwm2m.ota_pool_size:}") + private int otaPoolSize; @Getter @Value("${transport.lwm2m.clean_period_in_sec:}") private int cleanPeriodInSec; - @Getter - @Value("${transport.lwm2m.update_registered_pool_size:}") - private int updateRegisteredPoolSize; - - @Getter - @Value("${transport.lwm2m.un_registered_pool_size:}") - private int unRegisteredPoolSize; - @Getter @Value("${transport.lwm2m.security.key_store_type:}") private String keyStoreType; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java index 97ab82f802..9425ef7891 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java @@ -27,6 +27,7 @@ import org.eclipse.leshan.server.californium.LeshanServerBuilder; import org.eclipse.leshan.server.californium.registration.CaliforniumRegistrationStore; import org.eclipse.leshan.server.model.LwM2mModelProvider; import org.springframework.stereotype.Component; +import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; @@ -80,6 +81,7 @@ public class DefaultLwM2mTransportService implements LwM2MTransportService { private final LwM2mTransportContext context; private final LwM2MTransportServerConfig config; private final LwM2mTransportServerHelper helper; + private final OtaPackageDataCache otaPackageDataCache; private final DefaultLwM2MUplinkMsgHandler handler; private final CaliforniumRegistrationStore registrationStore; private final TbSecurityStore securityStore; @@ -103,8 +105,7 @@ public class DefaultLwM2mTransportService implements LwM2MTransportService { * "coap://host:port/{path}/{token}/{nameFile}" */ - - LwM2mTransportCoapResource otaCoapResource = new LwM2mTransportCoapResource(handler, FIRMWARE_UPDATE_COAP_RECOURSE); + LwM2mTransportCoapResource otaCoapResource = new LwM2mTransportCoapResource(otaPackageDataCache, FIRMWARE_UPDATE_COAP_RECOURSE); this.server.coap().getServer().add(otaCoapResource); this.startLhServer(); this.context.setServer(server); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java index 00f70dbcf7..4fc60aecfc 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java @@ -84,10 +84,10 @@ public class LwM2mNetworkConfig { Create new instance of udp endpoint context matcher. Params: checkAddress - – true with address check, (STRICT, UDP) + – true with address check, (STRICT, UDP) - if port Registration of client is changed - it is bad - false, without */ - coapConfig.setString(NetworkConfig.Keys.RESPONSE_MATCHING, "STRICT"); + coapConfig.setString(NetworkConfig.Keys.RESPONSE_MATCHING, "RELAXED"); /** https://tools.ietf.org/html/rfc7959#section-2.9.3 The block size (number of bytes) to use when doing a blockwise transfer. \ @@ -103,7 +103,7 @@ public class LwM2mNetworkConfig { */ coapConfig.setInt(NetworkConfig.Keys.MAX_MESSAGE_SIZE, 1024); - coapConfig.setInt(NetworkConfig.Keys.MAX_RETRANSMIT, 4); + coapConfig.setInt(NetworkConfig.Keys.MAX_RETRANSMIT, 10); return coapConfig; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java index ff86ee9bd9..229ec1a08c 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java @@ -24,6 +24,7 @@ import org.eclipse.californium.core.observe.ObserveRelation; import org.eclipse.californium.core.server.resources.CoapExchange; import org.eclipse.californium.core.server.resources.Resource; import org.eclipse.californium.core.server.resources.ResourceObserver; +import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; @@ -39,11 +40,11 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.S public class LwM2mTransportCoapResource extends AbstractLwM2mTransportResource { private final ConcurrentMap tokenToObserveRelationMap = new ConcurrentHashMap<>(); private final ConcurrentMap tokenToObserveNotificationSeqMap = new ConcurrentHashMap<>(); - private final LwM2mUplinkMsgHandler handler; + private final OtaPackageDataCache otaPackageDataCache; - public LwM2mTransportCoapResource(LwM2mUplinkMsgHandler handler, String name) { + public LwM2mTransportCoapResource(OtaPackageDataCache otaPackageDataCache, String name) { super(name); - this.handler = handler; + this.otaPackageDataCache = otaPackageDataCache; this.setObservable(true); // enable observing this.addObserver(new CoapResourceObserver()); } @@ -154,7 +155,7 @@ public class LwM2mTransportCoapResource extends AbstractLwM2mTransportResource { } private byte[] getOtaData(UUID currentId) { - return ((DefaultLwM2MUplinkMsgHandler) handler).otaPackageDataCache.get(currentId.toString()); + return otaPackageDataCache.get(currentId.toString()); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java index f310eb6481..d04e4993f1 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java @@ -201,36 +201,12 @@ public class LwM2mTransportUtil { } } - /** - * FirmwareUpdateStatus { - * DOWNLOADING, DOWNLOADED, VERIFIED, UPDATING, UPDATED, FAILED - */ - public static OtaPackageUpdateStatus equalsFwSateFwResultToFirmwareUpdateStatus(UpdateStateFw updateStateFw, UpdateResultFw updateResultFw) { - switch (updateResultFw) { - case INITIAL: - return toOtaPackageUpdateStatus(updateStateFw); - case UPDATE_SUCCESSFULLY: - return UPDATED; - case NOT_ENOUGH: - case OUT_OFF_MEMORY: - case CONNECTION_LOST: - case INTEGRITY_CHECK_FAILURE: - case UNSUPPORTED_TYPE: - case INVALID_URI: - case UPDATE_FAILED: - case UNSUPPORTED_PROTOCOL: - return FAILED; - default: - throw new CodecException("Invalid value stateFw %s %s for FirmwareUpdateStatus.", updateStateFw.name(), updateResultFw.name()); - } - } - - public static OtaPackageUpdateStatus toOtaPackageUpdateStatus(UpdateResultFw updateResultFw) { + public static Optional toOtaPackageUpdateStatus(UpdateResultFw updateResultFw) { switch (updateResultFw) { case INITIAL: - return VERIFIED; + return Optional.empty(); case UPDATE_SUCCESSFULLY: - return UPDATED; + return Optional.of(UPDATED); case NOT_ENOUGH: case OUT_OFF_MEMORY: case CONNECTION_LOST: @@ -239,22 +215,22 @@ public class LwM2mTransportUtil { case INVALID_URI: case UPDATE_FAILED: case UNSUPPORTED_PROTOCOL: - return FAILED; + return Optional.of(FAILED); default: throw new CodecException("Invalid value stateFw %s for FirmwareUpdateStatus.", updateResultFw.name()); } } - public static OtaPackageUpdateStatus toOtaPackageUpdateStatus(UpdateStateFw updateStateFw) { + public static Optional toOtaPackageUpdateStatus(UpdateStateFw updateStateFw) { switch (updateStateFw) { case IDLE: - return VERIFIED; + return Optional.empty(); case DOWNLOADING: - return DOWNLOADING; + return Optional.of(DOWNLOADING); case DOWNLOADED: - return DOWNLOADED; + return Optional.of(DOWNLOADED); case UPDATING: - return UPDATING; + return Optional.of(UPDATING); default: throw new CodecException("Invalid value stateFw %d for FirmwareUpdateStatus.", updateStateFw); } @@ -471,7 +447,7 @@ public class LwM2mTransportUtil { return lwM2mOtaConvert; } else if (FW_RESULT_ID.equals(path)) { lwM2mOtaConvert.setCurrentType(STRING); - lwM2mOtaConvert.setValue(UpdateResultFw.fromUpdateResultFwByCode(((Long) value).intValue()).type); + lwM2mOtaConvert.setValue(UpdateResultFw.fromUpdateResultFwByCode(((Long) value).intValue()).getType()); return lwM2mOtaConvert; } else if (SW_UPDATE_STATE_ID.equals(path)) { lwM2mOtaConvert.setCurrentType(STRING); @@ -585,7 +561,7 @@ public class LwM2mTransportUtil { public static String fromVersionedIdToObjectId(String pathIdVer) { try { - if(pathIdVer == null) { + if (pathIdVer == null) { return null; } String[] keyArray = pathIdVer.split(LWM2M_SEPARATOR_PATH); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java index 1b2e7d6b26..79361aeb1d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.transport.lwm2m.server; +import lombok.Getter; + /** * FW Update Result * 0: Initial value. Once the updating process is initiated (Download /Update), this Resource MUST be reset to Initial value. @@ -40,14 +42,17 @@ public enum UpdateResultFw { UPDATE_FAILED(8, "Firmware update failed", false), UNSUPPORTED_PROTOCOL(9, "Unsupported protocol", false); - public int code; - public String type; - public boolean isAgain; + @Getter + private int code; + @Getter + private String type; + @Getter + private boolean again; UpdateResultFw(int code, String type, boolean isAgain) { this.code = code; this.type = type; - this.isAgain = isAgain; + this.again = isAgain; } public static UpdateResultFw fromUpdateResultFwByType(String type) { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/DefaultLwM2MAttributesService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/DefaultLwM2MAttributesService.java index ed717f3eee..ee723dbaa6 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/DefaultLwM2MAttributesService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/attributes/DefaultLwM2MAttributesService.java @@ -39,6 +39,7 @@ import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; import org.thingsboard.server.transport.lwm2m.server.downlink.LwM2mDownlinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplaceRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService; import org.thingsboard.server.transport.lwm2m.server.ota.LwM2MOtaUpdateService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; @@ -71,6 +72,7 @@ public class DefaultLwM2MAttributesService implements LwM2MAttributesService { private final LwM2MTransportServerConfig config; private final LwM2mUplinkMsgHandler uplinkHandler; private final LwM2mDownlinkMsgHandler downlinkHandler; + private final LwM2MTelemetryLogService logService; private final LwM2MOtaUpdateService otaUpdateService; @Override @@ -150,14 +152,6 @@ public class DefaultLwM2MAttributesService implements LwM2MAttributesService { if (!otherAttributes.isEmpty()) { onAttributesUpdate(lwM2MClient, otherAttributes); } - } else if (msg.getSharedDeletedCount() > 0 && lwM2MClient != null) { - msg.getSharedUpdatedList().forEach(tsKvProto -> { - String pathName = tsKvProto.getKv().getKey(); - Object valueNew = getValueFromKvProto(tsKvProto.getKv()); - if (OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION).equals(pathName) && !valueNew.equals(lwM2MClient.getFwUpdate().getCurrentVersion())) { - lwM2MClient.getFwUpdate().setCurrentVersion((String) valueNew); - } - }); } else if (lwM2MClient == null) { log.error("OnAttributeUpdate, lwM2MClient is null"); } @@ -197,12 +191,12 @@ public class DefaultLwM2MAttributesService implements LwM2MAttributesService { private void pushUpdateToClientIfNeeded(LwM2mClient lwM2MClient, Object valueOld, Object newValue, String versionedId) { if (newValue != null && (valueOld == null || !newValue.toString().equals(valueOld.toString()))) { TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId).value(newValue).timeout(this.config.getTimeout()).build(); - downlinkHandler.sendWriteReplaceRequest(lwM2MClient, request, new TbLwM2MWriteResponseCallback(uplinkHandler, lwM2MClient, versionedId)); + downlinkHandler.sendWriteReplaceRequest(lwM2MClient, request, new TbLwM2MWriteResponseCallback(uplinkHandler, logService, lwM2MClient, versionedId)); } else { log.error("Failed update resource [{}] [{}]", versionedId, newValue); String logMsg = String.format("%s: Failed update resource versionedId - %s value - %s. Value is not changed or bad", LOG_LWM2M_ERROR, versionedId, newValue); - uplinkHandler.logToTelemetry(lwM2MClient, logMsg); + logService.log(lwM2MClient, logMsg); log.info("Failed update resource [{}] [{}]", versionedId, newValue); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java index 21d6ec3d48..0e71a1e865 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java @@ -95,12 +95,6 @@ public class LwM2mClient implements Cloneable { private UUID profileId; @Getter @Setter - private volatile LwM2mFwSwUpdate fwUpdate; - @Getter - @Setter - private volatile LwM2mFwSwUpdate swUpdate; - @Getter - @Setter private Registration registration; private ValidateDeviceCredentialsResponse credentials; @@ -365,21 +359,5 @@ public class LwM2mClient implements Cloneable { } } - public LwM2mFwSwUpdate getFwUpdate(LwM2mUplinkMsgHandler handler, LwM2mClientContext clientContext) { - if (this.fwUpdate == null) { - var profile = clientContext.getProfile(this.getProfileId()); - this.fwUpdate = new LwM2mFwSwUpdate(handler, this, OtaPackageType.FIRMWARE, profile.getClientLwM2mSettings().getFwUpdateStrategy()); - } - return this.fwUpdate; - } - - public LwM2mFwSwUpdate getSwUpdate(LwM2mUplinkMsgHandler handler, LwM2mClientContext clientContext) { - if (this.swUpdate == null) { - var profile = clientContext.getProfile(this.getProfileId()); - this.swUpdate = new LwM2mFwSwUpdate(handler, this, OtaPackageType.SOFTWARE, profile.getClientLwM2mSettings().getSwUpdateStrategy()); - } - return this.fwUpdate; - } - } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java deleted file mode 100644 index b39f6084cd..0000000000 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFwSwUpdate.java +++ /dev/null @@ -1,486 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.lwm2m.server.client; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.leshan.server.registration.Registration; -import org.thingsboard.server.common.data.ota.OtaPackageType; -import org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.LwM2mOperationType; -import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; -import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; -import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; -import org.thingsboard.server.transport.lwm2m.server.downlink.LwM2mDownlinkMsgHandler; -import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; -import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MExecuteRequest; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MExecuteCallback; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MObserveRequest; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MObserveCallback; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplaceRequest; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.CopyOnWriteArrayList; - -import static org.thingsboard.server.common.data.ota.OtaPackageKey.STATE; -import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; -import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.DOWNLOADED; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.FAILED; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.INITIATED; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATED; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATING; -import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getAttributeKey; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FIRMWARE_UPDATE_COAP_RECOURSE; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_3_VER_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_VER_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_NAME_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_PACKAGE_19_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_PACKAGE_5_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_PACKAGE_URI_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_RESULT_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_STATE_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_UPDATE; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_UPDATE_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_ERROR; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; -import static org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY; -import static org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; -import static org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mOperationType.EXECUTE; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mOperationType.WRITE_REPLACE; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_INSTALL_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_NAME_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_PACKAGE_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_RESULT_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_UN_INSTALL_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_UPDATE; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_UPDATE_STATE_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_VER_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.toOtaPackageUpdateStatus; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.splitCamelCaseString; - -@Slf4j -public class LwM2mFwSwUpdate { - // 5/0/6 PkgName - // 9/0/0 PkgName - @Getter - @Setter - private volatile String currentTitle; - // 5/0/7 PkgVersion - // 9/0/1 PkgVersion - @Getter - @Setter - private volatile String currentVersion; - @Getter - @Setter - private volatile UUID currentId; - @Getter - @Setter - private volatile String stateUpdate; - @Getter - private String pathPackageId; - @Getter - private String pathStateId; - @Getter - private String pathResultId; - @Getter - private String pathNameId; - @Getter - private String pathVerId; - @Getter - private String pathInstallId; - @Getter - private String pathUnInstallId; - @Getter - private String wUpdate; - @Getter - @Setter - private volatile boolean infoFwSwUpdate = false; - private final OtaPackageType type; - - private final LwM2mUplinkMsgHandler handler; - - @Getter - LwM2mClient lwM2MClient; - @Getter - private final List pendingInfoRequestsStart; - @Getter - @Setter - private volatile int updateStrategy; - - public LwM2mFwSwUpdate(LwM2mUplinkMsgHandler handler, LwM2mClient lwM2MClient, OtaPackageType type, int updateStrategy) { - this.handler = handler; - this.lwM2MClient = lwM2MClient; - this.pendingInfoRequestsStart = new CopyOnWriteArrayList<>(); - this.type = type; - this.stateUpdate = null; - this.updateStrategy = updateStrategy; - this.initPathId(); - } - - private void initPathId() { - if (FIRMWARE.equals(this.type)) { - this.pathPackageId = LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy ? - FW_PACKAGE_5_ID : LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == this.updateStrategy ? - FW_PACKAGE_URI_ID : FW_PACKAGE_19_ID; - this.pathStateId = FW_STATE_ID; - this.pathResultId = FW_RESULT_ID; - this.pathNameId = FW_NAME_ID; - this.pathVerId = FW_5_VER_ID; - this.pathInstallId = FW_UPDATE_ID; - this.wUpdate = FW_UPDATE; - } else if (SOFTWARE.equals(this.type)) { - this.pathPackageId = SW_PACKAGE_ID; - this.pathStateId = SW_UPDATE_STATE_ID; - this.pathResultId = SW_RESULT_ID; - this.pathNameId = SW_NAME_ID; - this.pathVerId = SW_VER_ID; - this.pathInstallId = SW_INSTALL_ID; - this.pathUnInstallId = SW_UN_INSTALL_ID; - this.wUpdate = SW_UPDATE; - } - } - - public void initReadValue(DefaultLwM2MUplinkMsgHandler handler, LwM2mDownlinkMsgHandler request, String pathIdVer) { - if (pathIdVer != null) { - this.pendingInfoRequestsStart.remove(pathIdVer); - } - if (this.pendingInfoRequestsStart.size() == 0) { - this.infoFwSwUpdate = false; -// if (!FAILED.name().equals(this.stateUpdate)) { - boolean conditionalStart = this.type.equals(FIRMWARE) ? this.conditionalFwUpdateStart(handler) : - this.conditionalSwUpdateStart(handler); - if (conditionalStart) { - this.writeFwSwWare(handler, request); - } -// } - } - } - - /** - * Send FsSw to Lwm2mClient: - * before operation Write: fw_state = DOWNLOADING - */ - public void writeFwSwWare(DefaultLwM2MUplinkMsgHandler handler, LwM2mDownlinkMsgHandler request) { - if (this.currentId != null) { - this.stateUpdate = OtaPackageUpdateStatus.INITIATED.name(); - this.sendLogs(handler, WRITE_REPLACE.name(), LOG_LWM2M_INFO, null); - String targetIdVer = convertObjectIdToVersionedId(this.pathPackageId, this.lwM2MClient.getRegistration()); - String fwMsg = String.format("%s: Start type operation %s paths: %s", LOG_LWM2M_INFO, - LwM2mOperationType.FW_UPDATE.name(), this.pathPackageId); - handler.logToTelemetry(fwMsg, lwM2MClient.getRegistration().getId()); - log.warn("8) Start firmware Update. Send save to: [{}] ver: [{}] path: [{}]", this.lwM2MClient.getDeviceName(), this.currentVersion, targetIdVer); - if (LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy) { - int chunkSize = 0; - int chunk = 0; - byte[] firmwareChunk = handler.otaPackageDataCache.get(this.currentId.toString(), chunkSize, chunk); - - TbLwM2MWriteReplaceRequest downlink = TbLwM2MWriteReplaceRequest.builder().versionedId(targetIdVer).value(firmwareChunk).timeout(handler.config.getTimeout()).build(); - request.sendWriteReplaceRequest(lwM2MClient, downlink, new TbLwM2MWriteResponseCallback(handler, lwM2MClient, targetIdVer)); - } else if (LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == this.updateStrategy) { - String apiFont = "coap://176.36.143.9:5685"; - String uri = apiFont + "/" + FIRMWARE_UPDATE_COAP_RECOURSE + "/" + this.currentId.toString(); - log.warn("89) coapUri: [{}]", uri); - //TODO: user this.rpcRequest??? - TbLwM2MWriteReplaceRequest downlink = TbLwM2MWriteReplaceRequest.builder().versionedId(targetIdVer).value(uri).timeout(handler.config.getTimeout()).build(); - request.sendWriteReplaceRequest(lwM2MClient, downlink, new TbLwM2MWriteResponseCallback(handler, lwM2MClient, targetIdVer)); - } else if (LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY.code == this.updateStrategy) { - - } - } else { - String msgError = "FirmWareId is null."; - log.warn("6) [{}]", msgError); -// if (this.rpcRequest != null) { -// TODO: refactor. -// handler.sentRpcResponse(this.rpcRequest, CONTENT.name(), msgError, LOG_LW2M_ERROR); -// } - log.error(msgError); - this.sendLogs(handler, WRITE_REPLACE.name(), LOG_LWM2M_ERROR, msgError); - } - } - - public void sendLogs(DefaultLwM2MUplinkMsgHandler handler, String typeOper, String typeInfo, String msgError) { -// this.sendSateOnThingsBoard(handler); - String msg = String.format("%s: %s, %s, pkgVer: %s: pkgName - %s state - %s.", - typeInfo, this.wUpdate, typeOper, this.currentVersion, this.currentTitle, this.stateUpdate); - if (LOG_LWM2M_ERROR.equals(typeInfo)) { - msg = String.format("%s Error: %s", msg, msgError); - } - handler.logToTelemetry(lwM2MClient, msg); - } - - - /** - * After inspection Update Result - * fw_state/sw_state = UPDATING - * send execute - */ - public void executeFwSwWare(DefaultLwM2MUplinkMsgHandler handler, LwM2mDownlinkMsgHandler request) { - this.sendLogs(handler, EXECUTE.name(), LOG_LWM2M_INFO, null); - //TODO: user this.rpcRequest??? - TbLwM2MExecuteRequest downlink = TbLwM2MExecuteRequest.builder().versionedId(pathInstallId).timeout(handler.config.getTimeout()).build(); - request.sendExecuteRequest(lwM2MClient, downlink, new TbLwM2MExecuteCallback(handler, lwM2MClient, pathInstallId)); - } - - /** - * Firmware start: Check if the version has changed and launch a new update. - * -ObjectId 5, Binary or ObjectId 5, URI - * -- If the result of the update - errors (more than 1) - This means that the previous. the update failed. - * - We launch the update regardless of the state of the firmware and its version. - * -- If the result of the update - errors (more than 1) - This means that the previous. the update failed. - * * ObjectId 5, Binary - * -- If the result of the update is not errors (equal to 1 or 0) and ver in Object 5 is not empty - it means that the previous update has passed. - * Compare current versions by equals. - * * ObjectId 5, URI - * -- If the result of the update is not errors (equal to 1 or 0) and ver in Object 5 is not empty - it means that the previous update has passed. - * Compare current versions by contains. - */ - private boolean conditionalFwUpdateStart(DefaultLwM2MUplinkMsgHandler handler) { - Long updateResultFw = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - String ver5 = (String) this.lwM2MClient.getResourceValue(null, this.pathVerId); - String pathName = (String) this.lwM2MClient.getResourceValue(null, this.pathNameId); - String ver3 = (String) this.lwM2MClient.getResourceValue(null, FW_3_VER_ID); - // #1/#2 - String fwMsg = null; - if ((this.currentVersion != null && ( - ver5 != null && ver5.equals(this.currentVersion) || - ver3 != null && ver3.contains(this.currentVersion) - )) || - (this.currentTitle != null && pathName != null && this.currentTitle.equals(pathName))) { - fwMsg = String.format("%s: The update was interrupted. The device has the same version: %s.", LOG_LWM2M_ERROR, - this.currentVersion); - } else if (updateResultFw != null && updateResultFw > UpdateResultFw.UPDATE_SUCCESSFULLY.code) { - fwMsg = String.format("%s: The update was interrupted. The device has the status UpdateResult: error (%d).", LOG_LWM2M_ERROR, - updateResultFw); - } - if (fwMsg != null) { - handler.logToTelemetry(fwMsg, lwM2MClient.getRegistration().getId()); - return false; - } else { - return true; - } - } - - - /** - * Before operation Execute inspection Update Result : - * 0 - Initial value - */ - public boolean conditionalFwExecuteStart() { - Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return UpdateResultFw.INITIAL.code == updateResult; - } - - /** - * After operation Execute success inspection Update Result : - * 1 - "Firmware updated successfully" - */ - public boolean conditionalFwExecuteAfterSuccess() { - Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return UpdateResultFw.UPDATE_SUCCESSFULLY.code == updateResult; - } - - /** - * After operation Execute success inspection Update Result : - * > 1 error: "Firmware updated successfully" - */ - public boolean conditionalFwExecuteAfterError() { - Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return UpdateResultFw.UPDATE_SUCCESSFULLY.code < updateResult; - } - - /** - * Software start - * - If Update Result -errors (equal or more than 50) - This means that the previous. the update failed. - * * - We launch the update regardless of the state of the firmware and its version. - * - If Update Result is not errors (less than 50) and ver is not empty - This means that before. the update has passed. - * - If Update Result is not errors and ver is empty - This means that there was no update yet or before. UnInstall update - * - If Update Result is not errors and ver is not empty - This means that before unInstall update - * * - Check if the version has changed and launch a new update. - */ - private boolean conditionalSwUpdateStart(DefaultLwM2MUplinkMsgHandler handler) { - Long updateResultSw = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - // #1/#2 - return updateResultSw >= LwM2mTransportUtil.UpdateResultSw.NOT_ENOUGH_STORAGE.code || - ( - (updateResultSw <= LwM2mTransportUtil.UpdateResultSw.NOT_ENOUGH_STORAGE.code - ) && - ( - (this.currentVersion != null && !this.currentVersion.equals(this.lwM2MClient.getResourceValue(null, this.pathVerId))) || - (this.currentTitle != null && !this.currentTitle.equals(this.lwM2MClient.getResourceValue(null, this.pathNameId))) - ) - ); - } - - /** - * Before operation Execute inspection Update Result : - * 3 - Successfully Downloaded and package integrity verified - */ - public boolean conditionalSwUpdateExecute() { - Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return LwM2mTransportUtil.UpdateResultSw.SUCCESSFULLY_DOWNLOADED_VERIFIED.code == updateResult; - } - - /** - * After finish operation Execute (success): - * -- inspection Update Result: - * ---- FW если Update Result == 1 ("Firmware updated successfully") или SW если Update Result == 2 ("Software successfully installed.") - * -- fw_state/sw_state = UPDATED - *

- * After finish operation Execute (error): - * -- inspection updateResult and send to thingsboard info about error - * --- send to telemetry ( key - this is name Update Result in model) ( - * -- fw_state/sw_state = FAILED - */ - public void finishFwSwUpdate(DefaultLwM2MUplinkMsgHandler handler, boolean success) { - Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - String value = FIRMWARE.equals(this.type) ? UpdateResultFw.fromUpdateResultFwByCode(updateResult.intValue()).type : - LwM2mTransportUtil.UpdateResultSw.fromUpdateResultSwByCode(updateResult.intValue()).type; - String key = splitCamelCaseString((String) this.lwM2MClient.getResourceNameByRezId(null, this.pathResultId)); - if (success) { - this.stateUpdate = OtaPackageUpdateStatus.UPDATED.name(); - this.sendLogs(handler, EXECUTE.name(), LOG_LWM2M_INFO, null); - } else { - this.stateUpdate = OtaPackageUpdateStatus.FAILED.name(); - this.sendLogs(handler, EXECUTE.name(), LOG_LWM2M_ERROR, value); - } - handler.helper.sendParametersOnThingsboardTelemetry( - handler.helper.getKvStringtoThingsboard(key, value), this.lwM2MClient.getSession()); - } - - /** - * After operation Execute success inspection Update Result : - * 2 - "Software successfully installed." - */ - public boolean conditionalSwExecuteAfterSuccess() { - Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return LwM2mTransportUtil.UpdateResultSw.SUCCESSFULLY_INSTALLED.code == updateResult; - } - - /** - * After operation Execute success inspection Update Result : - * >= 50 - error "NOT_ENOUGH_STORAGE" - */ - public boolean conditionalSwExecuteAfterError() { - Long updateResult = (Long) this.lwM2MClient.getResourceValue(null, this.pathResultId); - return LwM2mTransportUtil.UpdateResultSw.NOT_ENOUGH_STORAGE.code <= updateResult; - } - -// private void observeStateUpdate(DefaultLwM2MUplinkMsgHandler handler, LwM2mDownlinkMsgHandler request) { -// request.sendAllRequest(lwM2MClient, -// convertPathFromObjectIdToIdVer(this.pathStateId, this.lwM2MClient.getRegistration()), OBSERVE, -// null, null, 0, null); -// request.sendAllRequest(lwM2MClient, -// convertPathFromObjectIdToIdVer(this.pathResultId, this.lwM2MClient.getRegistration()), OBSERVE, -// null, null, 0, null); -// } - - public void sendSateOnThingsBoard(DefaultLwM2MUplinkMsgHandler handler) { - if (StringUtils.trimToNull(this.stateUpdate) != null) { - List result = new ArrayList<>(); - TransportProtos.KeyValueProto.Builder kvProto = TransportProtos.KeyValueProto.newBuilder().setKey(getAttributeKey(this.type, STATE)); - kvProto.setType(TransportProtos.KeyValueType.STRING_V).setStringV(stateUpdate); - result.add(kvProto.build()); - handler.helper.sendParametersOnThingsboardTelemetry(result, - handler.getSessionInfoOrCloseSession(this.lwM2MClient.getRegistration())); - } - } - - public void sendReadObserveInfo(LwM2mDownlinkMsgHandler request) { - this.infoFwSwUpdate = true; - this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( - this.pathStateId, this.lwM2MClient.getRegistration())); - this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( - this.pathResultId, this.lwM2MClient.getRegistration())); - this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( - FW_3_VER_ID, this.lwM2MClient.getRegistration())); - if (LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY.code == this.updateStrategy || - LwM2MFirmwareUpdateStrategy.OBJ_19_BINARY.code == this.updateStrategy || - SOFTWARE.equals(this.type)) { - this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( - this.pathVerId, this.lwM2MClient.getRegistration())); - this.pendingInfoRequestsStart.add(convertObjectIdToVersionedId( - this.pathNameId, this.lwM2MClient.getRegistration())); - } - this.pendingInfoRequestsStart.forEach(versionedId -> { - TbLwM2MObserveRequest downlink = TbLwM2MObserveRequest.builder().versionedId(versionedId).build(); - request.sendObserveRequest(this.lwM2MClient, downlink, new TbLwM2MObserveCallback(handler, lwM2MClient, versionedId)); - }); - - } - - /** - * Before operation Execute (FwUpdate) inspection Update Result : - * - after finished operation Write result: success (FwUpdate): fw_state = DOWNLOADED - * - before start operation Execute (FwUpdate) Update Result = 0 - Initial value - * - start Execute (FwUpdate) - * After finished operation Execute (FwUpdate) inspection Update Result : - * - after start operation Execute (FwUpdate): fw_state = UPDATING - * - after success finished operation Execute (FwUpdate) Update Result == 1 ("Firmware updated successfully") - * - finished operation Execute (FwUpdate) - */ - public void updateStateOta(DefaultLwM2MUplinkMsgHandler handler, LwM2mDownlinkMsgHandler request, - Registration registration, String path, int value) { - if (OBJ_5_BINARY.code == this.getUpdateStrategy()) { - if ((convertObjectIdToVersionedId(FW_RESULT_ID, registration).equals(path))) { - if (DOWNLOADED.name().equals(this.getStateUpdate()) - && this.conditionalFwExecuteStart()) { - this.executeFwSwWare(handler, request); - } else if (UPDATING.name().equals(this.getStateUpdate()) - && this.conditionalFwExecuteAfterSuccess()) { - this.finishFwSwUpdate(handler, true); - } else if (UPDATING.name().equals(this.getStateUpdate()) - && this.conditionalFwExecuteAfterError()) { - this.finishFwSwUpdate(handler, false); - } - } - } else if (OBJ_5_TEMP_URL.code == this.getUpdateStrategy()) { - if (this.currentId != null && (convertObjectIdToVersionedId(FW_STATE_ID, registration).equals(path))) { - String state = toOtaPackageUpdateStatus(UpdateStateFw.fromStateFwByCode(value)).name(); - if (StringUtils.isNotEmpty(state) && !FAILED.name().equals(this.stateUpdate) && !state.equals(this.stateUpdate)) { - this.stateUpdate = state; - this.sendSateOnThingsBoard(handler); - } - if (value == UpdateStateFw.DOWNLOADED.code) { - this.executeFwSwWare(handler, request); - } - handler.firmwareUpdateState.put(lwM2MClient.getEndpoint(), value); - } - if ((convertObjectIdToVersionedId(FW_RESULT_ID, registration).equals(path))) { - if (this.currentId != null && value == UpdateResultFw.INITIAL.code) { - this.setStateUpdate(INITIATED.name()); - } else if (this.currentId != null && value == UpdateResultFw.UPDATE_SUCCESSFULLY.code) { - this.setStateUpdate(UPDATED.name()); - } else if (value > UpdateResultFw.UPDATE_SUCCESSFULLY.code) { - this.setStateUpdate(FAILED.name()); - } - this.sendSateOnThingsBoard(handler); - } - } else if (OBJ_19_BINARY.code == this.getUpdateStrategy()) { - - } - } -} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/common/LwM2MExecutorAwareService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/common/LwM2MExecutorAwareService.java new file mode 100644 index 0000000000..1f1dff784f --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/common/LwM2MExecutorAwareService.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.common; + +import org.thingsboard.common.util.ThingsBoardExecutors; + +import javax.annotation.PreDestroy; +import java.util.concurrent.ExecutorService; + +public abstract class LwM2MExecutorAwareService { + + protected ExecutorService executor; + + protected abstract int getExecutorSize(); + + protected abstract String getExecutorName(); + + protected void init() { + this.executor = ThingsBoardExecutors.newWorkStealingPool(getExecutorSize(), getExecutorName()); + } + + public void destroy() { + if (executor != null) { + executor.shutdownNow(); + } + } + +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/AbstractTbLwM2MRequestCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/AbstractTbLwM2MRequestCallback.java index 764d5ff5ef..561b103277 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/AbstractTbLwM2MRequestCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/AbstractTbLwM2MRequestCallback.java @@ -17,30 +17,30 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; -import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_WARN; @Slf4j public abstract class AbstractTbLwM2MRequestCallback implements DownlinkRequestCallback { - protected final LwM2mUplinkMsgHandler handler; + protected final LwM2MTelemetryLogService logService; protected final LwM2mClient client; - protected AbstractTbLwM2MRequestCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client) { - this.handler = handler; + protected AbstractTbLwM2MRequestCallback(LwM2MTelemetryLogService logService, LwM2mClient client) { + this.logService = logService; this.client = client; } @Override public void onValidationError(String params, String msg) { log.trace("[{}] Request [{}] validation failed. Reason: {}", client.getEndpoint(), params, msg); - handler.logToTelemetry(client, String.format("[%s]: Request [%s] validation failed. Reason: %s", LOG_LWM2M_WARN, params, msg)); + logService.log(client, String.format("[%s]: Request [%s] validation failed. Reason: %s", LOG_LWM2M_WARN, params, msg)); } @Override public void onError(String params, Exception e) { log.trace("[{}] Request [{}] processing failed", client.getEndpoint(), params, e); - handler.logToTelemetry(client, String.format("[%s]: Request [%s] processing failed. Reason: %s", LOG_LWM2M_WARN, params, e)); + logService.log(client, String.format("[%s]: Request [%s] processing failed. Reason: %s", LOG_LWM2M_WARN, params, e)); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java index da513a1934..52ef83bf8e 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java @@ -26,7 +26,6 @@ import org.eclipse.leshan.core.node.LwM2mResource; import org.eclipse.leshan.core.node.ObjectLink; import org.eclipse.leshan.core.node.codec.CodecException; import org.eclipse.leshan.core.observation.Observation; -import org.eclipse.leshan.core.request.CancelObservationRequest; import org.eclipse.leshan.core.request.ContentFormat; import org.eclipse.leshan.core.request.DeleteRequest; import org.eclipse.leshan.core.request.DiscoverRequest; @@ -45,7 +44,6 @@ import org.eclipse.leshan.core.response.ReadResponse; import org.eclipse.leshan.core.response.WriteAttributesResponse; import org.eclipse.leshan.core.response.WriteResponse; import org.eclipse.leshan.core.util.Hex; -import org.eclipse.leshan.core.util.NamedThreadFactory; import org.eclipse.leshan.server.registration.Registration; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -54,17 +52,18 @@ import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportContext; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import org.thingsboard.server.transport.lwm2m.server.common.LwM2MExecutorAwareService; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.utils.LwM2mValueConverterImpl; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -74,25 +73,38 @@ import static org.eclipse.leshan.core.attributes.Attribute.LESSER_THAN; import static org.eclipse.leshan.core.attributes.Attribute.MAXIMUM_PERIOD; import static org.eclipse.leshan.core.attributes.Attribute.MINIMUM_PERIOD; import static org.eclipse.leshan.core.attributes.Attribute.STEP; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.RESPONSE_REQUEST_CHANNEL; @Slf4j @Service @TbLwM2mTransportComponent @RequiredArgsConstructor -public class DefaultLwM2mDownlinkMsgHandler implements LwM2mDownlinkMsgHandler { - private ExecutorService responseRequestExecutor; +public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService implements LwM2mDownlinkMsgHandler { public LwM2mValueConverterImpl converter; private final LwM2mTransportContext context; private final LwM2MTransportServerConfig config; + private final LwM2MTelemetryLogService logService; @PostConstruct public void init() { + super.init(); this.converter = LwM2mValueConverterImpl.getInstance(); - responseRequestExecutor = Executors.newFixedThreadPool(this.config.getResponsePoolSize(), - new NamedThreadFactory(String.format("LwM2M %s channel response after request", RESPONSE_REQUEST_CHANNEL))); + } + + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + protected int getExecutorSize() { + return config.getDownlinkPoolSize(); + } + + @Override + protected String getExecutorName() { + return "LwM2M Downlink"; } @Override @@ -221,7 +233,8 @@ public class DefaultLwM2mDownlinkMsgHandler implements LwM2mDownlinkMsgHandler { **/ Collection resources = client.getNewResourceForInstance(request.getVersionedId(), request.getValue(), this.config.getModelProvider(), this.converter); ResourceModel resourceModelWrite = client.getResourceModel(request.getVersionedId(), this.config.getModelProvider()); - WriteRequest downlink = new WriteRequest(WriteRequest.Mode.UPDATE, convertResourceModelTypeToContentFormat(client, resourceModelWrite.type), resultIds.getObjectId(), + ContentFormat contentFormat = request.getObjectContentFormat() != null ? request.getObjectContentFormat() : convertResourceModelTypeToContentFormat(client, resourceModelWrite.type); + WriteRequest downlink = new WriteRequest(WriteRequest.Mode.UPDATE, contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId(), resources); sendRequest(client, downlink, request.getTimeout(), callback); } else if (resultIds.isObjectInstance()) { @@ -245,19 +258,24 @@ public class DefaultLwM2mDownlinkMsgHandler implements LwM2mDownlinkMsgHandler { private , T extends LwM2mResponse> void sendRequest(LwM2mClient client, R request, long timeoutInMs, DownlinkRequestCallback callback) { Registration registration = client.getRegistration(); - context.getServer().send(registration, request, timeoutInMs, response -> { - responseRequestExecutor.submit(() -> { - try { - callback.onSuccess(request, response); - } catch (Exception e) { - log.error("[{}] failed to process successful response [{}] ", registration.getEndpoint(), response, e); - } + try { + logService.log(client, String.format("[%s][%s] Sending request: %s to %s", registration.getId(), registration.getSocketAddress(), request.getClass().getSimpleName(), request.getPath())); + context.getServer().send(registration, request, timeoutInMs, response -> { + executor.submit(() -> { + try { + callback.onSuccess(request, response); + } catch (Exception e) { + log.error("[{}] failed to process successful response [{}] ", registration.getEndpoint(), response, e); + } + }); + }, e -> { + executor.submit(() -> { + callback.onError(JacksonUtil.toString(request), e); + }); }); - }, e -> { - responseRequestExecutor.submit(() -> { - callback.onError(JacksonUtil.toString(request), e); - }); - }); + } catch (Exception e) { + callback.onError(JacksonUtil.toString(request), e); + } } private WriteRequest getWriteRequestSingleResource(ResourceModel.Type type, ContentFormat contentFormat, int objectId, int instanceId, int resourceId, Object value) { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelAllObserveCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelAllObserveCallback.java index 42f2a1baec..f90e0f8b97 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelAllObserveCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelAllObserveCallback.java @@ -17,21 +17,21 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; -import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; @Slf4j public class TbLwM2MCancelAllObserveCallback extends AbstractTbLwM2MRequestCallback { - public TbLwM2MCancelAllObserveCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client) { - super(handler, client); + public TbLwM2MCancelAllObserveCallback(LwM2MTelemetryLogService logService, LwM2mClient client) { + super(logService, client); } @Override public void onSuccess(TbLwM2MCancelAllRequest request, Integer canceledSubscriptionsCount) { log.trace("[{}] Cancel of all observations was successful: {}", client.getEndpoint(), canceledSubscriptionsCount); - handler.logToTelemetry(client, String.format("[%s]: Cancel of all observations was successful. Result: [%s]", LOG_LWM2M_INFO, canceledSubscriptionsCount)); + logService.log(client, String.format("[%s]: Cancel of all observations was successful. Result: [%s]", LOG_LWM2M_INFO, canceledSubscriptionsCount)); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelObserveCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelObserveCallback.java index eec33963d5..311be03462 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelObserveCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MCancelObserveCallback.java @@ -16,6 +16,7 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; @@ -27,15 +28,15 @@ public class TbLwM2MCancelObserveCallback extends AbstractTbLwM2MRequestCallback private final String versionedId; - public TbLwM2MCancelObserveCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String versionedId) { - super(handler, client); + public TbLwM2MCancelObserveCallback(LwM2MTelemetryLogService logService, LwM2mClient client, String versionedId) { + super(logService, client); this.versionedId = versionedId; } @Override public void onSuccess(TbLwM2MCancelObserveRequest request, Integer canceledSubscriptionsCount) { log.trace("[{}] Cancel observation of [{}] successful: {}", client.getEndpoint(), versionedId, canceledSubscriptionsCount); - handler.logToTelemetry(client, String.format("[%s]: Cancel Observe for [%s] successful. Result: [%s]", LOG_LWM2M_INFO, versionedId, canceledSubscriptionsCount)); + logService.log(client, String.format("[%s]: Cancel Observe for [%s] successful. Result: [%s]", LOG_LWM2M_INFO, versionedId, canceledSubscriptionsCount)); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDeleteCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDeleteCallback.java index 32c82404c6..4c0a758572 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDeleteCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDeleteCallback.java @@ -18,12 +18,13 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import org.eclipse.leshan.core.request.DeleteRequest; import org.eclipse.leshan.core.response.DeleteResponse; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; public class TbLwM2MDeleteCallback extends TbLwM2MTargetedCallback { - public TbLwM2MDeleteCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String targetId) { - super(handler, client, targetId); + public TbLwM2MDeleteCallback(LwM2MTelemetryLogService logService, LwM2mClient client, String targetId) { + super(logService, client, targetId); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDiscoverCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDiscoverCallback.java index 5ae20e064d..41172f0599 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDiscoverCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MDiscoverCallback.java @@ -17,13 +17,14 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import org.eclipse.leshan.core.request.DiscoverRequest; import org.eclipse.leshan.core.response.DiscoverResponse; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; public class TbLwM2MDiscoverCallback extends TbLwM2MTargetedCallback { - public TbLwM2MDiscoverCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String targetId) { - super(handler, client, targetId); + public TbLwM2MDiscoverCallback(LwM2MTelemetryLogService logService, LwM2mClient client, String targetId) { + super(logService, client, targetId); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MExecuteCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MExecuteCallback.java index d2aed6a4fd..992cf6d616 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MExecuteCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MExecuteCallback.java @@ -17,13 +17,14 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import org.eclipse.leshan.core.request.ExecuteRequest; import org.eclipse.leshan.core.response.ExecuteResponse; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; public class TbLwM2MExecuteCallback extends TbLwM2MTargetedCallback { - public TbLwM2MExecuteCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String targetId) { - super(handler, client, targetId); + public TbLwM2MExecuteCallback(LwM2MTelemetryLogService logService, LwM2mClient client, String targetId) { + super(logService, client, targetId); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MObserveCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MObserveCallback.java index 7fca29aa8c..dde49c870d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MObserveCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MObserveCallback.java @@ -18,14 +18,15 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.request.ObserveRequest; import org.eclipse.leshan.core.response.ObserveResponse; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; @Slf4j -public class TbLwM2MObserveCallback extends TbLwM2MTargetedCallback { +public class TbLwM2MObserveCallback extends TbLwM2MUplinkTargetedCallback { - public TbLwM2MObserveCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String targetId) { - super(handler, client, targetId); + public TbLwM2MObserveCallback(LwM2mUplinkMsgHandler handler, LwM2MTelemetryLogService logService, LwM2mClient client, String targetId) { + super(handler, logService, client, targetId); } @Override diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MReadCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MReadCallback.java index 657fd04fd0..59247b103b 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MReadCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MReadCallback.java @@ -19,13 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.request.ReadRequest; import org.eclipse.leshan.core.response.ReadResponse; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; @Slf4j -public class TbLwM2MReadCallback extends TbLwM2MTargetedCallback { +public class TbLwM2MReadCallback extends TbLwM2MUplinkTargetedCallback { - public TbLwM2MReadCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String targetId) { - super(handler, client, targetId); + public TbLwM2MReadCallback(LwM2mUplinkMsgHandler handler, LwM2MTelemetryLogService logService, LwM2mClient client, String targetId) { + super(handler, logService, client, targetId); } @Override diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MTargetedCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MTargetedCallback.java index 258bec53ac..373c882ec4 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MTargetedCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MTargetedCallback.java @@ -17,6 +17,7 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; @@ -26,8 +27,8 @@ public abstract class TbLwM2MTargetedCallback extends AbstractTbLwM2MReque protected final String versionedId; - public TbLwM2MTargetedCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String versionedId) { - super(handler, client); + public TbLwM2MTargetedCallback(LwM2MTelemetryLogService logService, LwM2mClient client, String versionedId) { + super(logService, client); this.versionedId = versionedId; } @@ -36,7 +37,7 @@ public abstract class TbLwM2MTargetedCallback extends AbstractTbLwM2MReque //TODO convert camelCase to "camel case" using .split("(? extends TbLwM2MTargetedCallback { + + protected LwM2mUplinkMsgHandler handler; + + public TbLwM2MUplinkTargetedCallback(LwM2mUplinkMsgHandler handler, LwM2MTelemetryLogService logService, LwM2mClient client, String versionedId) { + super(logService, client, versionedId); + this.handler = handler; + } + +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteAttributesCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteAttributesCallback.java index 372534fa7a..e346ad965a 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteAttributesCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteAttributesCallback.java @@ -18,12 +18,13 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import org.eclipse.leshan.core.request.WriteAttributesRequest; import org.eclipse.leshan.core.response.WriteAttributesResponse; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; public class TbLwM2MWriteAttributesCallback extends TbLwM2MTargetedCallback { - public TbLwM2MWriteAttributesCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String targetId) { - super(handler, client, targetId); + public TbLwM2MWriteAttributesCallback(LwM2MTelemetryLogService logService, LwM2mClient client, String targetId) { + super(logService, client, targetId); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteResponseCallback.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteResponseCallback.java index 89586bd113..4746077f19 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteResponseCallback.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/TbLwM2MWriteResponseCallback.java @@ -18,12 +18,13 @@ package org.thingsboard.server.transport.lwm2m.server.downlink; import org.eclipse.leshan.core.request.WriteRequest; import org.eclipse.leshan.core.response.WriteResponse; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; -public class TbLwM2MWriteResponseCallback extends TbLwM2MTargetedCallback { +public class TbLwM2MWriteResponseCallback extends TbLwM2MUplinkTargetedCallback { - public TbLwM2MWriteResponseCallback(LwM2mUplinkMsgHandler handler, LwM2mClient client, String targetId) { - super(handler, client, targetId); + public TbLwM2MWriteResponseCallback(LwM2mUplinkMsgHandler handler, LwM2MTelemetryLogService logService, LwM2mClient client, String targetId) { + super(handler, logService, client, targetId); } @Override diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/DefaultLwM2MTelemetryLogService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/DefaultLwM2MTelemetryLogService.java new file mode 100644 index 0000000000..421f4cf3a6 --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/DefaultLwM2MTelemetryLogService.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.log; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; +import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; + +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_TELEMETRY; + +@Slf4j +@Service +@TbLwM2mTransportComponent +@RequiredArgsConstructor +public class DefaultLwM2MTelemetryLogService implements LwM2MTelemetryLogService { + + private final LwM2mClientContext clientContext; + private final LwM2mTransportServerHelper helper; + + /** + * @param logMsg - text msg + * @param registrationId - Id of Registration LwM2M Client + */ + @Override + public void log(String registrationId, String logMsg) { + log(clientContext.getClientByRegistrationId(registrationId), logMsg); + } + + @Override + public void log(LwM2mClient client, String logMsg) { + if (logMsg != null && client != null && client.getSession() != null) { + if (logMsg.length() > 1024) { + logMsg = logMsg.substring(0, 1024); + } + this.helper.sendParametersOnThingsboardTelemetry(this.helper.getKvStringtoThingsboard(LOG_LWM2M_TELEMETRY, logMsg), client.getSession()); + } + } + +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/LwM2MTelemetryLogService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/LwM2MTelemetryLogService.java new file mode 100644 index 0000000000..ff8543303e --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/log/LwM2MTelemetryLogService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.log; + +import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; + +public interface LwM2MTelemetryLogService { + + void log(LwM2mClient client, String msg); + + void log(String registrationId, String msg); + +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java index 3f86131320..d44770e591 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java @@ -22,7 +22,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageKey; @@ -41,33 +40,35 @@ import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; import org.thingsboard.server.transport.lwm2m.server.attributes.LwM2MAttributesService; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; +import org.thingsboard.server.transport.lwm2m.server.common.LwM2MExecutorAwareService; import org.thingsboard.server.transport.lwm2m.server.downlink.LwM2mDownlinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MExecuteCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MExecuteRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplaceRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; -import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteUpdateRequest; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; import static org.thingsboard.server.common.data.ota.OtaPackageKey.STATE; import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getAttributeKey; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FIRMWARE_UPDATE_COAP_RECOURSE; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_TELEMETRY; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; @Slf4j @Service @TbLwM2mTransportComponent @RequiredArgsConstructor -public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { +public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService implements LwM2MOtaUpdateService { public static final String FIRMWARE_VERSION = getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION); public static final String FIRMWARE_TITLE = getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.TITLE); @@ -93,8 +94,8 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { private final LwM2mUplinkMsgHandler uplinkHandler; private final LwM2mDownlinkMsgHandler downlinkHandler; private final OtaPackageDataCache otaPackageDataCache; + private final LwM2MTelemetryLogService logService; private final LwM2mTransportServerHelper helper; - private ExecutorService executor; @Autowired @Lazy @@ -102,8 +103,22 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { @PostConstruct public void init() { - //TODO: define parallelism in constant - executor = ThingsBoardExecutors.newWorkStealingPool(4, "LwM2M OTA Updates"); + super.init(); + } + + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + protected int getExecutorSize() { + return config.getOtaPoolSize(); + } + + @Override + protected String getExecutorName() { + return "LwM2M OTA"; } @Override @@ -112,7 +127,6 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { //TODO: check that the client supports FW and SW by checking the supported objects in the model. List attributesToFetch = new ArrayList<>(); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); - fwInfo.setSupported(client.isValidObjectVersion(FW_NAME_ID) || client.isValidObjectVersion(FW_VER_ID)); if (fwInfo.isSupported()) { attributesToFetch.add(FIRMWARE_TITLE); attributesToFetch.add(FIRMWARE_VERSION); @@ -138,6 +152,14 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { } } + @Override + public void forceFirmwareUpdate(LwM2mClient client) { + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.setRetryAttempts(0); + fwInfo.setFailedPackageId(null); + startFirmwareUpdateIfNeeded(client, fwInfo); + } + @Override public void onTargetFirmwareUpdate(LwM2mClient client, String newFirmwareTitle, String newFirmwareVersion, Optional newFirmwareUrl) { LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); @@ -147,33 +169,48 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { @Override public void onCurrentFirmwareNameUpdate(LwM2mClient client, String name) { + log.debug("[{}] Current fw name: {}", client.getEndpoint(), name); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); fwInfo.setCurrentName(name); } @Override - public void onCurrentFirmwareVersionUpdate(LwM2mClient client, String version) { + public void onCurrentFirmwareVersion3Update(LwM2mClient client, String version) { + log.debug("[{}] Current fw version: {}", client.getEndpoint(), version); + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.setCurrentVersion3(version); + } + + @Override + public void onCurrentFirmwareVersion5Update(LwM2mClient client, String version) { + log.debug("[{}] Current fw version: {}", client.getEndpoint(), version); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); - fwInfo.setCurrentVersion(version); + fwInfo.setCurrentVersion5(version); } @Override - public void onCurrentFirmwareStateUpdate(LwM2mClient client, Long state) { + public void onCurrentFirmwareStateUpdate(LwM2mClient client, Long stateCode) { + log.debug("[{}] Current fw state: {}", client.getEndpoint(), stateCode); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); - UpdateStateFw newState = UpdateStateFw.fromStateFwByCode(state.intValue()); - if (UpdateStateFw.DOWNLOADED.equals(newState)) { + UpdateStateFw state = UpdateStateFw.fromStateFwByCode(stateCode.intValue()); + if (UpdateStateFw.DOWNLOADED.equals(state)) { executeFwUpdate(client); } - fwInfo.setUpdateState(newState); - sendStateUpdateToTelemetry(client, fwInfo, LwM2mTransportUtil.toOtaPackageUpdateStatus(newState)); + fwInfo.setUpdateState(state); + Optional status = LwM2mTransportUtil.toOtaPackageUpdateStatus(state); + status.ifPresent(otaStatus -> sendStateUpdateToTelemetry(client, fwInfo, + otaStatus, "Firmware Update State: " + state.name())); } @Override public void onCurrentFirmwareResultUpdate(LwM2mClient client, Long code) { + log.debug("[{}] Current fw result: {}", client.getEndpoint(), code); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); UpdateResultFw result = UpdateResultFw.fromUpdateResultFwByCode(code.intValue()); - sendStateUpdateToTelemetry(client, fwInfo, LwM2mTransportUtil.toOtaPackageUpdateStatus(result)); - if (result.isAgain && fwInfo.getRetryAttempts() <= 2) { + Optional status = LwM2mTransportUtil.toOtaPackageUpdateStatus(result); + status.ifPresent(otaStatus -> sendStateUpdateToTelemetry(client, fwInfo, + otaStatus, "Firmware Update Result: " + result.name())); + if (result.isAgain() && fwInfo.getRetryAttempts() <= 2) { fwInfo.setRetryAttempts(fwInfo.getRetryAttempts() + 1); startFirmwareUpdateIfNeeded(client, fwInfo); } else { @@ -183,6 +220,7 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { @Override public void onCurrentFirmwareDeliveryMethodUpdate(LwM2mClient client, Long value) { + log.debug("[{}] Current fw delivery method: {}", client.getEndpoint(), value); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); fwInfo.setDeliveryMethod(value.intValue()); } @@ -193,22 +231,29 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { } private void startFirmwareUpdateIfNeeded(LwM2mClient client, LwM2MClientOtaInfo fwInfo) { - if (fwInfo.isUpdateRequired()) { - if (StringUtils.isNotEmpty(fwInfo.getTargetUrl())) { - log.info("[{}] Starting update to [{}{}] using URL: {}", client.getEndpoint(), fwInfo.getTargetName(), fwInfo.getTargetVersion(), fwInfo.getTargetUrl()); - startFirmwareUpdateUsingUrl(client, fwInfo.getTargetUrl()); - } else { - startFirmwareUpdateUsingBinary(client, fwInfo); + try { + if (!fwInfo.isSupported()) { + log.debug("[{}] Fw update is not supported: {}", client.getEndpoint(), fwInfo); + sendStateUpdateToTelemetry(client, fwInfo, OtaPackageUpdateStatus.FAILED, "Client does not support firmware update or profile misconfiguration!"); + } else if (fwInfo.isUpdateRequired()) { + if (StringUtils.isNotEmpty(fwInfo.getTargetUrl())) { + log.debug("[{}] Starting update to [{}{}] using URL: {}", client.getEndpoint(), fwInfo.getTargetName(), fwInfo.getTargetVersion(), fwInfo.getTargetUrl()); + startFirmwareUpdateUsingUrl(client, fwInfo.getTargetUrl()); + } else { + log.debug("[{}] Starting update to [{}{}] using binary", client.getEndpoint(), fwInfo.getTargetName(), fwInfo.getTargetVersion()); + startFirmwareUpdateUsingBinary(client, fwInfo); + } } - } else if (fwInfo.isUpdateFailed()) { - sendStateUpdateToTelemetry(client, fwInfo, OtaPackageUpdateStatus.FAILED); + } catch (Exception e) { + log.warn("[{}] failed to update client: {}", client.getEndpoint(), fwInfo, e); + sendStateUpdateToTelemetry(client, fwInfo, OtaPackageUpdateStatus.FAILED, "Internal server error: " + e.getMessage()); } } private void startFirmwareUpdateUsingUrl(LwM2mClient client, String url) { String targetIdVer = convertObjectIdToVersionedId(FW_URL_ID, client.getRegistration()); - TbLwM2MWriteUpdateRequest request = TbLwM2MWriteUpdateRequest.builder().versionedId(targetIdVer).value(url).timeout(config.getTimeout()).build(); - downlinkHandler.sendWriteUpdateRequest(client, request, new TbLwM2MWriteResponseCallback(uplinkHandler, client, targetIdVer)); + TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(targetIdVer).value(url).timeout(config.getTimeout()).build(); + downlinkHandler.sendWriteReplaceRequest(client, request, new TbLwM2MWriteResponseCallback(uplinkHandler, logService, client, targetIdVer)); } public void startFirmwareUpdateUsingBinary(LwM2mClient client, LwM2MClientOtaInfo fwInfo) { @@ -217,41 +262,44 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { new TransportServiceCallback<>() { @Override public void onSuccess(TransportProtos.GetOtaPackageResponseMsg response) { - if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus()) - && response.getType().equals(OtaPackageType.FIRMWARE.name())) { - UUID otaPackageId = new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB()); - LwM2MFirmwareUpdateStrategy strategy; - if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == 2) { - strategy = fwInfo.getStrategy(); - } else { - strategy = fwInfo.getDeliveryMethod() == 0 ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; - } - switch (strategy) { - case OBJ_5_BINARY: - byte[] firmwareChunk = otaPackageDataCache.get(otaPackageId.toString(), 0, 0); - TbLwM2MWriteReplaceRequest writeRequest = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId) - .value(firmwareChunk).contentFormat(ContentFormat.OPAQUE) - .timeout(config.getTimeout()).build(); - downlinkHandler.sendWriteReplaceRequest(client, writeRequest, new TbLwM2MWriteResponseCallback(uplinkHandler, client, versionedId)); - break; - case OBJ_5_TEMP_URL: - startFirmwareUpdateUsingUrl(client, fwInfo.getBaseUrl() + "/" + FIRMWARE_UPDATE_COAP_RECOURSE + "/" + otaPackageId.toString()); - break; - default: - //TODO: send log to telemetry - } - } else { - //TODO: send log to telemetry - } + executor.submit(() -> doUpdateFirmwareUsingBinary(response, fwInfo, versionedId, client)); } @Override public void onError(Throwable e) { - log.trace("Failed to process firmwareUpdate ", e); + logService.log(client, "Failed to process firmware update: " + e.getMessage()); } }); } + private void doUpdateFirmwareUsingBinary(TransportProtos.GetOtaPackageResponseMsg response, LwM2MClientOtaInfo fwInfo, String versionedId, LwM2mClient client) { + if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus())) { + UUID otaPackageId = new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB()); + LwM2MFirmwareUpdateStrategy strategy; + if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == 2) { + strategy = fwInfo.getStrategy(); + } else { + strategy = fwInfo.getDeliveryMethod() == 0 ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; + } + switch (strategy) { + case OBJ_5_BINARY: + byte[] firmwareChunk = otaPackageDataCache.get(otaPackageId.toString(), 0, 0); + TbLwM2MWriteReplaceRequest writeRequest = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId) + .value(firmwareChunk).contentFormat(ContentFormat.OPAQUE) + .timeout(config.getTimeout()).build(); + downlinkHandler.sendWriteReplaceRequest(client, writeRequest, new TbLwM2MWriteResponseCallback(uplinkHandler, logService, client, versionedId)); + break; + case OBJ_5_TEMP_URL: + startFirmwareUpdateUsingUrl(client, fwInfo.getBaseUrl() + "/" + FIRMWARE_UPDATE_COAP_RECOURSE + "/" + otaPackageId.toString()); + break; + default: + sendStateUpdateToTelemetry(client, fwInfo, OtaPackageUpdateStatus.FAILED, "Unsupported strategy: " + strategy.name()); + } + } else { + sendStateUpdateToTelemetry(client, fwInfo, OtaPackageUpdateStatus.FAILED, "Failed to fetch OTA package: " + response.getResponseStatus()); + } + } + private TransportProtos.GetOtaPackageRequestMsg createOtaPackageRequestMsg(TransportProtos.SessionInfoProto sessionInfo, String nameFwSW) { return TransportProtos.GetOtaPackageRequestMsg.newBuilder() .setDeviceIdMSB(sessionInfo.getDeviceIdMSB()) @@ -264,7 +312,7 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { private void executeFwUpdate(LwM2mClient client) { TbLwM2MExecuteRequest request = TbLwM2MExecuteRequest.builder().versionedId(FW_EXECUTE_ID).timeout(config.getTimeout()).build(); - downlinkHandler.sendExecuteRequest(client, request, new TbLwM2MExecuteCallback(uplinkHandler, client, FW_EXECUTE_ID)); + downlinkHandler.sendExecuteRequest(client, request, new TbLwM2MExecuteCallback(logService, client, FW_EXECUTE_ID)); } private Optional getAttributeValue(List attrs, String keyName) { @@ -298,11 +346,14 @@ public class DefaultLwM2MOtaUpdateService implements LwM2MOtaUpdateService { } - private void sendStateUpdateToTelemetry(LwM2mClient client, LwM2MClientOtaInfo fwInfo, OtaPackageUpdateStatus status) { + private void sendStateUpdateToTelemetry(LwM2mClient client, LwM2MClientOtaInfo fwInfo, OtaPackageUpdateStatus status, String log) { List result = new ArrayList<>(); TransportProtos.KeyValueProto.Builder kvProto = TransportProtos.KeyValueProto.newBuilder().setKey(getAttributeKey(fwInfo.getType(), STATE)); kvProto.setType(TransportProtos.KeyValueType.STRING_V).setStringV(status.name()); result.add(kvProto.build()); + kvProto = TransportProtos.KeyValueProto.newBuilder().setKey(LOG_LWM2M_TELEMETRY); + kvProto.setType(TransportProtos.KeyValueType.STRING_V).setStringV(log); + result.add(kvProto.build()); helper.sendParametersOnThingsboardTelemetry(result, client.getSession()); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java index 50b44f8d06..43d4f4acf5 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java @@ -31,7 +31,6 @@ public class LwM2MClientOtaInfo { private final OtaPackageType type; private String baseUrl; - private boolean supported; private boolean targetFetchFailure; private String targetName; @@ -40,7 +39,8 @@ public class LwM2MClientOtaInfo { private boolean currentFetchFailure; private String currentName; - private String currentVersion; + private String currentVersion3; + private String currentVersion5; private Integer deliveryMethod; //TODO: use value from device if applicable; @@ -65,24 +65,27 @@ public class LwM2MClientOtaInfo { } public boolean isUpdateRequired() { - if (StringUtils.isEmpty(targetName) || - StringUtils.isEmpty(targetVersion) || - (StringUtils.isEmpty(currentName) && StringUtils.isEmpty(currentVersion))) { + if (StringUtils.isEmpty(targetName) || StringUtils.isEmpty(targetVersion) || !isSupported()) { return false; } else { String targetPackageId = getPackageId(targetName, targetVersion); - String currentPackageId = getPackageId(currentName, currentVersion); + String currentPackageIdUsingObject5 = getPackageId(currentName, currentVersion5); if (StringUtils.isNotEmpty(failedPackageId) && failedPackageId.equals(targetPackageId)) { return false; } else { - return !targetPackageId.equals(currentPackageId); + if (targetPackageId.equals(currentPackageIdUsingObject5)) { + return false; + } else if (StringUtils.isNotEmpty(currentVersion3)) { + return !currentVersion3.contains(targetPackageId); + } else { + return true; + } } } } - public boolean isUpdateFailed() { - return (StringUtils.isNotEmpty(targetName) && StringUtils.isNotEmpty(targetVersion) && - (StringUtils.isEmpty(currentName) && StringUtils.isEmpty(currentVersion))); + public boolean isSupported() { + return StringUtils.isNotEmpty(currentName) || StringUtils.isNotEmpty(currentVersion5) || StringUtils.isNotEmpty(currentVersion3); } public void setUpdateResult(UpdateResultFw updateResult) { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java index 6386e94364..9c7905a5e8 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java @@ -23,13 +23,17 @@ public interface LwM2MOtaUpdateService { void init(LwM2mClient client); + void forceFirmwareUpdate(LwM2mClient client); + void onTargetFirmwareUpdate(LwM2mClient client, String newFirmwareTitle, String newFirmwareVersion, Optional newFirmwareUrl); void onTargetSoftwareUpdate(LwM2mClient client, String newSoftwareTitle, String newSoftwareVersion); void onCurrentFirmwareNameUpdate(LwM2mClient client, String name); - void onCurrentFirmwareVersionUpdate(LwM2mClient client, String version); + void onCurrentFirmwareVersion3Update(LwM2mClient client, String version); + + void onCurrentFirmwareVersion5Update(LwM2mClient client, String version); void onCurrentFirmwareStateUpdate(LwM2mClient client, Long state); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java index 533458ce76..5de223fa2f 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java @@ -50,6 +50,7 @@ import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteAttrib import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplaceRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteUpdateRequest; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import java.util.Map; @@ -69,6 +70,7 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { private final LwM2MTransportServerConfig config; private final LwM2mUplinkMsgHandler uplinkHandler; private final LwM2mDownlinkMsgHandler downlinkHandler; + private final LwM2MTelemetryLogService logService; private final Map rpcSubscriptions = new ConcurrentHashMap<>(); @Override @@ -148,14 +150,14 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { private void sendReadRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { TbLwM2MReadRequest request = TbLwM2MReadRequest.builder().versionedId(versionedId).timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MReadCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MReadCallback(uplinkHandler, logService, client, versionedId); var rpcCallback = new RpcReadResponseCallback<>(transportService, client, requestMsg, versionedId, mainCallback); downlinkHandler.sendReadRequest(client, request, rpcCallback); } private void sendObserveRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { TbLwM2MObserveRequest request = TbLwM2MObserveRequest.builder().versionedId(versionedId).timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MObserveCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MObserveCallback(uplinkHandler, logService, client, versionedId); var rpcCallback = new RpcReadResponseCallback<>(transportService, client, requestMsg, versionedId, mainCallback); downlinkHandler.sendObserveRequest(client, request, rpcCallback); } @@ -172,14 +174,14 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { private void sendDiscoverRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { TbLwM2MDiscoverRequest request = TbLwM2MDiscoverRequest.builder().versionedId(versionedId).timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MDiscoverCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MDiscoverCallback(logService, client, versionedId); var rpcCallback = new RpcDiscoverCallback(transportService, client, requestMsg, mainCallback); downlinkHandler.sendDiscoverRequest(client, request, rpcCallback); } private void sendExecuteRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { TbLwM2MExecuteRequest downlink = TbLwM2MExecuteRequest.builder().versionedId(versionedId).timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MExecuteCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MExecuteCallback(logService, client, versionedId); var rpcCallback = new RpcEmptyResponseCallback<>(transportService, client, requestMsg, mainCallback); downlinkHandler.sendExecuteRequest(client, downlink, rpcCallback); } @@ -189,7 +191,7 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { TbLwM2MWriteAttributesRequest request = TbLwM2MWriteAttributesRequest.builder().versionedId(versionedId) .attributes(requestBody.getAttributes()) .timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MWriteAttributesCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MWriteAttributesCallback(logService, client, versionedId); var rpcCallback = new RpcEmptyResponseCallback<>(transportService, client, requestMsg, mainCallback); downlinkHandler.sendWriteAttributesRequest(client, request, rpcCallback); } @@ -198,7 +200,7 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { RpcWriteUpdateRequest requestBody = JacksonUtil.fromString(requestMsg.getParams(), RpcWriteUpdateRequest.class); TbLwM2MWriteUpdateRequest.TbLwM2MWriteUpdateRequestBuilder builder = TbLwM2MWriteUpdateRequest.builder().versionedId(versionedId); builder.value(requestBody.getValue()).timeout(this.config.getTimeout()); - var mainCallback = new TbLwM2MWriteResponseCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MWriteResponseCallback(uplinkHandler, logService, client, versionedId); var rpcCallback = new RpcEmptyResponseCallback<>(transportService, client, requestMsg, mainCallback); downlinkHandler.sendWriteUpdateRequest(client, builder.build(), rpcCallback); } @@ -208,28 +210,28 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId) .value(requestBody.getValue()) .timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MWriteResponseCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MWriteResponseCallback(uplinkHandler, logService, client, versionedId); var rpcCallback = new RpcEmptyResponseCallback<>(transportService, client, requestMsg, mainCallback); downlinkHandler.sendWriteReplaceRequest(client, request, rpcCallback); } private void sendCancelObserveRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { TbLwM2MCancelObserveRequest downlink = TbLwM2MCancelObserveRequest.builder().versionedId(versionedId).timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MCancelObserveCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MCancelObserveCallback(logService, client, versionedId); var rpcCallback = new RpcCancelObserveCallback(transportService, client, requestMsg, mainCallback); downlinkHandler.sendCancelObserveRequest(client, downlink, rpcCallback); } private void sendDeleteRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { TbLwM2MDeleteRequest downlink = TbLwM2MDeleteRequest.builder().versionedId(versionedId).timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MDeleteCallback(uplinkHandler, client, versionedId); + var mainCallback = new TbLwM2MDeleteCallback(logService, client, versionedId); var rpcCallback = new RpcEmptyResponseCallback<>(transportService, client, requestMsg, mainCallback); downlinkHandler.sendDeleteRequest(client, downlink, rpcCallback); } private void sendCancelAllObserveRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg) { TbLwM2MCancelAllRequest downlink = TbLwM2MCancelAllRequest.builder().timeout(this.config.getTimeout()).build(); - var mainCallback = new TbLwM2MCancelAllObserveCallback(uplinkHandler, client); + var mainCallback = new TbLwM2MCancelAllObserveCallback(logService, client); var rpcCallback = new RpcCancelAllObserveCallback(transportService, client, requestMsg, mainCallback); downlinkHandler.sendCancelAllRequest(client, downlink, rpcCallback); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 61c4b077d9..5b61ffc768 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -44,8 +44,6 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.device.data.lwm2m.ObjectAttributes; import org.thingsboard.server.common.data.device.data.lwm2m.TelemetryMappingConfiguration; import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageUtil; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; @@ -66,9 +64,9 @@ import org.thingsboard.server.transport.lwm2m.server.client.LwM2MClientState; import org.thingsboard.server.transport.lwm2m.server.client.LwM2MClientStateException; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; -import org.thingsboard.server.transport.lwm2m.server.client.LwM2mFwSwUpdate; import org.thingsboard.server.transport.lwm2m.server.client.ParametersAnalyzeResult; import org.thingsboard.server.transport.lwm2m.server.client.ResultsAddKeyValueProto; +import org.thingsboard.server.transport.lwm2m.server.common.LwM2MExecutorAwareService; import org.thingsboard.server.transport.lwm2m.server.downlink.DownlinkRequestCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.LwM2mDownlinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MCancelObserveCallback; @@ -82,12 +80,14 @@ import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MReadCallbac import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MReadRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteAttributesCallback; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteAttributesRequest; +import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.ota.LwM2MOtaUpdateService; import org.thingsboard.server.transport.lwm2m.server.rpc.LwM2MRpcRequestHandler; import org.thingsboard.server.transport.lwm2m.server.store.TbLwM2MDtlsSessionStore; import org.thingsboard.server.transport.lwm2m.utils.LwM2mValueConverterImpl; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -105,9 +105,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.FAILED; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_3_VER_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_VER_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_DELIVERY_METHOD; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_NAME_ID; @@ -115,22 +113,17 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.F import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_STATE_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_ERROR; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_TELEMETRY; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_WARN; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SW_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertOtaUpdateValueToString; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertOtaUpdateValueToString; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.fromVersionedIdToObjectId; @Slf4j @Service @TbLwM2mTransportComponent -public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { +public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService implements LwM2mUplinkMsgHandler { - private ExecutorService registrationExecutor; - private ExecutorService updateRegistrationExecutor; - private ExecutorService unRegistrationExecutor; public LwM2mValueConverterImpl converter; private final TransportService transportService; @@ -138,6 +131,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { private final LwM2MAttributesService attributesService; private final LwM2MOtaUpdateService otaService; public final LwM2MTransportServerConfig config; + private final LwM2MTelemetryLogService logService; public final OtaPackageDataCache otaPackageDataCache; public final LwM2mTransportServerHelper helper; private final TbLwM2MDtlsSessionStore sessionStore; @@ -148,8 +142,10 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { public final Map firmwareUpdateState; public DefaultLwM2MUplinkMsgHandler(TransportService transportService, - LwM2MTransportServerConfig config, LwM2mTransportServerHelper helper, + LwM2MTransportServerConfig config, + LwM2mTransportServerHelper helper, LwM2mClientContext clientContext, + LwM2MTelemetryLogService logService, @Lazy LwM2MOtaUpdateService otaService, @Lazy LwM2MAttributesService attributesService, @Lazy LwM2MRpcRequestHandler rpcHandler, @@ -162,6 +158,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { this.config = config; this.helper = helper; this.clientContext = clientContext; + this.logService = logService; this.rpcHandler = rpcHandler; this.defaultLwM2MDownlinkMsgHandler = defaultLwM2MDownlinkMsgHandler; this.otaPackageDataCache = otaPackageDataCache; @@ -171,14 +168,27 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { } @PostConstruct - public void initAttributes() { + public void init() { + super.init(); this.context.getScheduler().scheduleAtFixedRate(this::reportActivity, new Random().nextInt((int) config.getSessionReportTimeout()), config.getSessionReportTimeout(), TimeUnit.MILLISECONDS); - this.registrationExecutor = ThingsBoardExecutors.newWorkStealingPool(this.config.getRegisteredPoolSize(), "LwM2M registration"); - this.updateRegistrationExecutor = ThingsBoardExecutors.newWorkStealingPool(this.config.getUpdateRegisteredPoolSize(), "LwM2M update registration"); - this.unRegistrationExecutor = ThingsBoardExecutors.newWorkStealingPool(this.config.getUnRegisteredPoolSize(), "LwM2M unRegistration"); this.converter = LwM2mValueConverterImpl.getInstance(); } + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + protected String getExecutorName() { + return "LwM2M uplink"; + } + + @Override + protected int getExecutorSize() { + return config.getUplinkPoolSize(); + } + /** * Start registration device * Create session: Map, LwM2MClient> @@ -194,7 +204,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { * @param previousObservations - may be null */ public void onRegistered(Registration registration, Collection previousObservations) { - registrationExecutor.submit(() -> { + executor.submit(() -> { LwM2mClient lwM2MClient = this.clientContext.getClientByEndpoint(registration.getEndpoint()); try { log.warn("[{}] [{{}] Client: create after Registration", registration.getEndpoint(), registration.getId()); @@ -204,7 +214,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { log.info("[{}] Closing old session: {}", registration.getEndpoint(), new UUID(oldSessionInfo.get().getSessionIdMSB(), oldSessionInfo.get().getSessionIdLSB())); closeSession(oldSessionInfo.get()); } - this.logToTelemetry(lwM2MClient, LOG_LWM2M_INFO + ": Client registered with registration id: " + registration.getId()); + logService.log(lwM2MClient, LOG_LWM2M_INFO + ": Client registered with registration id: " + registration.getId()); SessionInfoProto sessionInfo = lwM2MClient.getSession(); transportService.registerAsyncSession(sessionInfo, new LwM2mSessionMsgListener(this, attributesService, rpcHandler, sessionInfo)); log.warn("40) sessionId [{}] Registering rpc subscription after Registration client", new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB())); @@ -215,8 +225,6 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { .setSubscribeToRPC(TransportProtos.SubscribeToRPCMsg.newBuilder().setSessionType(TransportProtos.SessionType.ASYNC).build()) .build(); transportService.process(msg, null); - this.getInfoFirmwareUpdate(lwM2MClient); - this.getInfoSoftwareUpdate(lwM2MClient); this.initClientTelemetry(lwM2MClient); this.initAttributes(lwM2MClient); otaService.init(lwM2MClient); @@ -229,11 +237,11 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { // Race condition detected and the client was in progress of unregistration while new registration arrived. Let's try again. onRegistered(registration, previousObservations); } else { - this.logToTelemetry(lwM2MClient, LOG_LWM2M_WARN + ": Client registration failed due to invalid state: " + stateException.getState()); + logService.log(lwM2MClient, LOG_LWM2M_WARN + ": Client registration failed due to invalid state: " + stateException.getState()); } } catch (Throwable t) { log.error("[{}] endpoint [{}] error Unable registration.", registration.getEndpoint(), t); - this.logToTelemetry(lwM2MClient, LOG_LWM2M_WARN + ": Client registration failed due to: " + t.getMessage()); + logService.log(lwM2MClient, LOG_LWM2M_WARN + ": Client registration failed due to: " + t.getMessage()); } }); } @@ -244,7 +252,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { * @param registration - Registration LwM2M Client */ public void updatedReg(Registration registration) { - updateRegistrationExecutor.submit(() -> { + executor.submit(() -> { LwM2mClient lwM2MClient = clientContext.getClientByEndpoint(registration.getEndpoint()); try { log.warn("[{}] [{{}] Client: update after Registration", registration.getEndpoint(), registration.getId()); @@ -265,7 +273,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { } } catch (Throwable t) { log.error("[{}] endpoint [{}] error Unable update registration.", registration.getEndpoint(), t); - this.logToTelemetry(lwM2MClient, LOG_LWM2M_ERROR + String.format(": Client update Registration, %s", t.getMessage())); + logService.log(lwM2MClient, LOG_LWM2M_ERROR + String.format(": Client update Registration, %s", t.getMessage())); } }); } @@ -275,10 +283,10 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { * @param observations - !!! Warn: if have not finishing unReg, then this operation will be finished on next Client`s connect */ public void unReg(Registration registration, Collection observations) { - unRegistrationExecutor.submit(() -> { + executor.submit(() -> { LwM2mClient client = clientContext.getClientByEndpoint(registration.getEndpoint()); try { - this.logToTelemetry(client, LOG_LWM2M_INFO + ": Client unRegistration"); + logService.log(client, LOG_LWM2M_INFO + ": Client unRegistration"); clientContext.unregister(client, registration); SessionInfoProto sessionInfo = client.getSession(); if (sessionInfo != null) { @@ -292,7 +300,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { log.info("[{}] delete registration: [{}] {}.", registration.getEndpoint(), stateException.getState(), stateException.getMessage()); } catch (Throwable t) { log.error("[{}] endpoint [{}] error Unable un registration.", registration.getEndpoint(), t); - this.logToTelemetry(client, LOG_LWM2M_ERROR + String.format(": Client Unable un Registration, %s", t.getMessage())); + logService.log(client, LOG_LWM2M_ERROR + String.format(": Client Unable un Registration, %s", t.getMessage())); } }); } @@ -305,7 +313,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { @Override public void onSleepingDev(Registration registration) { log.info("[{}] [{}] Received endpoint Sleeping version event", registration.getId(), registration.getEndpoint()); - this.logToTelemetry(clientContext.getClientByEndpoint(registration.getEndpoint()), LOG_LWM2M_INFO + ": Client is sleeping!"); + logService.log(clientContext.getClientByEndpoint(registration.getEndpoint()), LOG_LWM2M_INFO + ": Client is sleeping!"); //TODO: associate endpointId with device information. } @@ -403,29 +411,10 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { @Override public void onAwakeDev(Registration registration) { log.trace("[{}] [{}] Received endpoint Awake version event", registration.getId(), registration.getEndpoint()); - this.logToTelemetry(clientContext.getClientByEndpoint(registration.getEndpoint()), LOG_LWM2M_INFO + ": Client is awake!"); + logService.log(clientContext.getClientByEndpoint(registration.getEndpoint()), LOG_LWM2M_INFO + ": Client is awake!"); //TODO: associate endpointId with device information. } - /** - * @param logMsg - text msg - * @param registrationId - Id of Registration LwM2M Client - */ - @Override - public void logToTelemetry(String registrationId, String logMsg) { - logToTelemetry(clientContext.getClientByRegistrationId(registrationId), logMsg); - } - - @Override - public void logToTelemetry(LwM2mClient client, String logMsg) { - if (logMsg != null && client != null && client.getSession() != null) { - if (logMsg.length() > 1024) { - logMsg = logMsg.substring(0, 1024); - } - this.helper.sendParametersOnThingsboardTelemetry(this.helper.getKvStringtoThingsboard(LOG_LWM2M_TELEMETRY, logMsg), client.getSession()); - } - } - /** * #1 clientOnlyObserveAfterConnect == true * - Only Observe Request to the client marked as observe from the profile configuration. @@ -441,10 +430,6 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2MClient.getProfileId()); Set supportedObjects = clientContext.getSupportedIdVerInClient(lwM2MClient); if (supportedObjects != null && supportedObjects.size() > 0) { - if (LwM2mTransportUtil.LwM2MClientStrategy.CLIENT_STRATEGY_2.code == profile.getClientLwM2mSettings().getClientOnlyObserveAfterConnect()) { - // #2 - supportedObjects.forEach(versionedId -> sendReadRequest(lwM2MClient, versionedId)); - } // #1 this.sendReadRequests(lwM2MClient, profile, supportedObjects); this.sendObserveRequests(lwM2MClient, profile, supportedObjects); @@ -461,7 +446,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { CountDownLatch latch = new CountDownLatch(targetIds.size()); targetIds.forEach(versionedId -> sendReadRequest(lwM2MClient, versionedId, - new TbLwM2MLatchCallback<>(latch, new TbLwM2MReadCallback(this, lwM2MClient, versionedId)))); + new TbLwM2MLatchCallback<>(latch, new TbLwM2MReadCallback(this, logService, lwM2MClient, versionedId)))); try { latch.await(); } catch (InterruptedException e) { @@ -475,7 +460,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { CountDownLatch latch = new CountDownLatch(targetIds.size()); targetIds.forEach(targetId -> sendObserveRequest(lwM2MClient, targetId, - new TbLwM2MLatchCallback<>(latch, new TbLwM2MObserveCallback(this, lwM2MClient, targetId)))); + new TbLwM2MLatchCallback<>(latch, new TbLwM2MObserveCallback(this, logService, lwM2MClient, targetId)))); try { latch.await(); } catch (InterruptedException e) { @@ -501,11 +486,11 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { private void sendDiscoverRequest(LwM2mClient lwM2MClient, String targetId) { TbLwM2MDiscoverRequest request = TbLwM2MDiscoverRequest.builder().versionedId(targetId).timeout(this.config.getTimeout()).build(); - defaultLwM2MDownlinkMsgHandler.sendDiscoverRequest(lwM2MClient, request, new TbLwM2MDiscoverCallback(this, lwM2MClient, targetId)); + defaultLwM2MDownlinkMsgHandler.sendDiscoverRequest(lwM2MClient, request, new TbLwM2MDiscoverCallback(logService, lwM2MClient, targetId)); } private void sendReadRequest(LwM2mClient lwM2MClient, String versionedId) { - sendReadRequest(lwM2MClient, versionedId, new TbLwM2MReadCallback(this, lwM2MClient, versionedId)); + sendReadRequest(lwM2MClient, versionedId, new TbLwM2MReadCallback(this, logService, lwM2MClient, versionedId)); } private void sendReadRequest(LwM2mClient lwM2MClient, String versionedId, DownlinkRequestCallback callback) { @@ -514,7 +499,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { } private void sendObserveRequest(LwM2mClient lwM2MClient, String versionedId) { - sendObserveRequest(lwM2MClient, versionedId, new TbLwM2MObserveCallback(this, lwM2MClient, versionedId)); + sendObserveRequest(lwM2MClient, versionedId, new TbLwM2MObserveCallback(this, logService, lwM2MClient, versionedId)); } private void sendObserveRequest(LwM2mClient lwM2MClient, String versionedId, DownlinkRequestCallback callback) { @@ -524,12 +509,12 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { private void sendWriteAttributesRequest(LwM2mClient lwM2MClient, String targetId, ObjectAttributes params) { TbLwM2MWriteAttributesRequest request = TbLwM2MWriteAttributesRequest.builder().versionedId(targetId).attributes(params).timeout(this.config.getTimeout()).build(); - defaultLwM2MDownlinkMsgHandler.sendWriteAttributesRequest(lwM2MClient, request, new TbLwM2MWriteAttributesCallback(this, lwM2MClient, targetId)); + defaultLwM2MDownlinkMsgHandler.sendWriteAttributesRequest(lwM2MClient, request, new TbLwM2MWriteAttributesCallback(logService, lwM2MClient, targetId)); } private void sendCancelObserveRequest(String versionedId, LwM2mClient client) { TbLwM2MCancelObserveRequest request = TbLwM2MCancelObserveRequest.builder().versionedId(versionedId).timeout(this.config.getTimeout()).build(); - defaultLwM2MDownlinkMsgHandler.sendCancelObserveRequest(client, request, new TbLwM2MCancelObserveCallback(this, client, versionedId)); + defaultLwM2MDownlinkMsgHandler.sendCancelObserveRequest(client, request, new TbLwM2MCancelObserveCallback(logService, client, versionedId)); } private void updateObjectResourceValue(LwM2mClient client, LwM2mObject lwM2mObject, String pathIdVer) { @@ -565,9 +550,9 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { if (path.equals(convertObjectIdToVersionedId(FW_NAME_ID, registration))) { otaService.onCurrentFirmwareNameUpdate(lwM2MClient, (String) lwM2mResource.getValue()); } else if (path.equals(convertObjectIdToVersionedId(FW_3_VER_ID, registration))) { - otaService.onCurrentFirmwareVersionUpdate(lwM2MClient, (String) lwM2mResource.getValue()); + otaService.onCurrentFirmwareVersion3Update(lwM2MClient, (String) lwM2mResource.getValue()); } else if (path.equals(convertObjectIdToVersionedId(FW_5_VER_ID, registration))) { - otaService.onCurrentFirmwareVersionUpdate(lwM2MClient, (String) lwM2mResource.getValue()); + otaService.onCurrentFirmwareVersion5Update(lwM2MClient, (String) lwM2mResource.getValue()); } else if (path.equals(convertObjectIdToVersionedId(FW_STATE_ID, registration))) { otaService.onCurrentFirmwareStateUpdate(lwM2MClient, (Long) lwM2mResource.getValue()); } else if (path.equals(convertObjectIdToVersionedId(FW_RESULT_ID, registration))) { @@ -919,93 +904,7 @@ public class DefaultLwM2MUplinkMsgHandler implements LwM2mUplinkMsgHandler { DonAsynchron.withCallback(attributesService.getSharedAttributes(lwM2MClient, keysToFetch), v -> attributesService.onAttributesUpdate(lwM2MClient, v), t -> log.error("[{}] Failed to get attributes", lwM2MClient.getEndpoint(), t), - registrationExecutor); - } - } - - public void getInfoFirmwareUpdate(LwM2mClient lwM2MClient) { - if (lwM2MClient.getRegistration().getSupportedVersion(FW_5_ID) != null) { - SessionInfoProto sessionInfo = this.getSessionInfo(lwM2MClient); - if (sessionInfo != null) { - DefaultLwM2MUplinkMsgHandler handler = this; - this.transportService.process(sessionInfo, createOtaPackageRequestMsg(sessionInfo, OtaPackageType.FIRMWARE.name()), - new TransportServiceCallback<>() { - @Override - public void onSuccess(TransportProtos.GetOtaPackageResponseMsg response) { - if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus()) - && response.getType().equals(OtaPackageType.FIRMWARE.name())) { - LwM2mFwSwUpdate fwUpdate = lwM2MClient.getFwUpdate(DefaultLwM2MUplinkMsgHandler.this, clientContext); -// if (rpcRequest != null) { -// fwUpdate.setStateUpdate(INITIATED.name()); -// } - if (!FAILED.name().equals(fwUpdate.getStateUpdate())) { - log.warn("7) firmware start with ver: [{}]", response.getVersion()); -// fwUpdate.setRpcRequest(rpcRequest); - fwUpdate.setCurrentVersion(response.getVersion()); - fwUpdate.setCurrentTitle(response.getTitle()); - fwUpdate.setCurrentId(new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB())); -// if (rpcRequest == null) { - fwUpdate.sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); -// } else { -// fwUpdate.writeFwSwWare(handler, defaultLwM2MDownlinkMsgHandler); -// } - } else { - String msgError = String.format("OtaPackage device: %s, version: %s, stateUpdate: %s", - lwM2MClient.getDeviceName(), response.getVersion(), fwUpdate.getStateUpdate()); - log.warn("7_1 [{}]", msgError); - } - } else { - String msgError = String.format("OtaPackage device: %s, responseStatus: %s", - lwM2MClient.getDeviceName(), response.getResponseStatus().toString()); - log.trace(msgError); -// if (rpcRequest != null) { - //TODO: refactor -// sendErrorRpcResponse(rpcRequest, msgError, sessionInfo); -// } - } - } - - @Override - public void onError(Throwable e) { - log.trace("Failed to process firmwareUpdate ", e); - } - }); - } - } - } - - public void getInfoSoftwareUpdate(LwM2mClient lwM2MClient) { - if (lwM2MClient.getRegistration().getSupportedVersion(SW_ID) != null) { - SessionInfoProto sessionInfo = this.getSessionInfo(lwM2MClient); - if (sessionInfo != null) { - DefaultLwM2MUplinkMsgHandler handler = this; - transportService.process(sessionInfo, createOtaPackageRequestMsg(sessionInfo, OtaPackageType.SOFTWARE.name()), - new TransportServiceCallback<>() { - @Override - public void onSuccess(TransportProtos.GetOtaPackageResponseMsg response) { - if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus()) - && response.getType().equals(OtaPackageType.SOFTWARE.name())) { -// lwM2MClient.getSwUpdate().setRpcRequest(rpcRequest); - lwM2MClient.getSwUpdate().setCurrentVersion(response.getVersion()); - lwM2MClient.getSwUpdate().setCurrentTitle(response.getTitle()); - lwM2MClient.getSwUpdate().setCurrentId(new OtaPackageId(new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB())).getId()); - lwM2MClient.getSwUpdate().sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); -// if (rpcRequest == null) { - lwM2MClient.getSwUpdate().sendReadObserveInfo(defaultLwM2MDownlinkMsgHandler); -// } else { -// lwM2MClient.getSwUpdate().writeFwSwWare(handler, defaultLwM2MDownlinkMsgHandler); -// } - } else { - log.trace("Software [{}] [{}]", lwM2MClient.getDeviceName(), response.getResponseStatus().toString()); - } - } - - @Override - public void onError(Throwable e) { - log.trace("Failed to process softwareUpdate ", e); - } - }); - } + executor); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java index 517db388e4..372daa7052 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mUplinkMsgHandler.java @@ -52,10 +52,6 @@ public interface LwM2mUplinkMsgHandler { void onAwakeDev(Registration registration); - void logToTelemetry(LwM2mClient client, String msg); - - void logToTelemetry(String registrationId, String msg); - void onWriteResponseOk(LwM2mClient client, String path, WriteRequest request); void onToTransportUpdateCredentials(TransportProtos.ToTransportUpdateCredentialsProto updateCredentials); diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index d81b28a5ff..d7e97231e0 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -142,12 +142,10 @@ transport: timeout: "${LWM2M_TIMEOUT:120000}" recommended_ciphers: "${LWM2M_RECOMMENDED_CIPHERS:false}" recommended_supported_groups: "${LWM2M_RECOMMENDED_SUPPORTED_GROUPS:true}" - response_pool_size: "${LWM2M_RESPONSE_POOL_SIZE:100}" - registered_pool_size: "${LWM2M_REGISTERED_POOL_SIZE:10}" - registration_store_pool_size: "${LWM2M_REGISTRATION_STORE_POOL_SIZE:100}" + uplink_pool_size: "${LWM2M_UPLINK_POOL_SIZE:10}" + downlink_pool_size: "${LWM2M_DOWNLINK_POOL_SIZE:10}" + ota_pool_size: "${LWM2M_OTA_POOL_SIZE:10}" clean_period_in_sec: "${LWM2M_CLEAN_PERIOD_IN_SEC:2}" - update_registered_pool_size: "${LWM2M_UPDATE_REGISTERED_POOL_SIZE:10}" - un_registered_pool_size: "${LWM2M_UN_REGISTERED_POOL_SIZE:10}" log_max_length: "${LWM2M_LOG_MAX_LENGTH:100}" # Use redis for Security and Registration stores redis.enabled: "${LWM2M_REDIS_ENABLED:false}" From 8b285c8e77f5ff6fb9b88e68c4acc375fa268b5a Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 15 Jun 2021 12:34:25 +0300 Subject: [PATCH 089/101] created OtaPackageTransportResource --- .../common/msg/session/FeatureType.java | 2 +- .../common/msg/session/SessionMsgType.java | 5 +- .../transport/coap/CoapTransportResource.java | 72 --------- .../transport/coap/CoapTransportService.java | 3 + .../coap/OtaPackageTransportResource.java | 144 ++++++++++++++++++ 5 files changed, 149 insertions(+), 77 deletions(-) create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java index ae965e327a..9be2957401 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/session/FeatureType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.common.msg.session; public enum FeatureType { - ATTRIBUTES, TELEMETRY, RPC, CLAIM, PROVISION, FIRMWARE, SOFTWARE + ATTRIBUTES, TELEMETRY, RPC, CLAIM, PROVISION } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/session/SessionMsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/session/SessionMsgType.java index 939197af80..5dbea04d59 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/session/SessionMsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/session/SessionMsgType.java @@ -30,10 +30,7 @@ public enum SessionMsgType { SESSION_OPEN, SESSION_CLOSE, - CLAIM_REQUEST(), - - GET_FIRMWARE_REQUEST, - GET_SOFTWARE_REQUEST; + CLAIM_REQUEST(); private final boolean requiresRulesProcessing; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 33f122fa3f..95cdeada88 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportC import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.security.DeviceTokenCredentials; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.common.msg.session.SessionMsgType; @@ -139,10 +138,6 @@ public class CoapTransportResource extends AbstractCoapTransportResource { processExchangeGetRequest(exchange, featureType.get()); } else if (featureType.get() == FeatureType.ATTRIBUTES) { processRequest(exchange, SessionMsgType.GET_ATTRIBUTES_REQUEST); - } else if (featureType.get() == FeatureType.FIRMWARE) { - processRequest(exchange, SessionMsgType.GET_FIRMWARE_REQUEST); - } else if (featureType.get() == FeatureType.SOFTWARE) { - processRequest(exchange, SessionMsgType.GET_SOFTWARE_REQUEST); } else { log.trace("Invalid feature type parameter"); exchange.respond(CoAP.ResponseCode.BAD_REQUEST); @@ -349,12 +344,6 @@ public class CoapTransportResource extends AbstractCoapTransportResource { coapTransportAdaptor.convertToGetAttributes(sessionId, request), new CoapNoOpCallback(exchange)); break; - case GET_FIRMWARE_REQUEST: - getOtaPackageCallback(sessionInfo, exchange, OtaPackageType.FIRMWARE); - break; - case GET_SOFTWARE_REQUEST: - getOtaPackageCallback(sessionInfo, exchange, OtaPackageType.SOFTWARE); - break; } } catch (AdaptorException e) { log.trace("[{}] Failed to decode message: ", sessionId, e); @@ -366,16 +355,6 @@ public class CoapTransportResource extends AbstractCoapTransportResource { return new UUID(sessionInfoProto.getSessionIdMSB(), sessionInfoProto.getSessionIdLSB()); } - private void getOtaPackageCallback(TransportProtos.SessionInfoProto sessionInfo, CoapExchange exchange, OtaPackageType firmwareType) { - TransportProtos.GetOtaPackageRequestMsg requestMsg = TransportProtos.GetOtaPackageRequestMsg.newBuilder() - .setTenantIdMSB(sessionInfo.getTenantIdMSB()) - .setTenantIdLSB(sessionInfo.getTenantIdLSB()) - .setDeviceIdMSB(sessionInfo.getDeviceIdMSB()) - .setDeviceIdLSB(sessionInfo.getDeviceIdLSB()) - .setType(firmwareType.name()).build(); - transportContext.getTransportService().process(sessionInfo, requestMsg, new OtaPackageCallback(exchange)); - } - private TransportProtos.SessionInfoProto lookupAsyncSessionInfo(String token) { tokenToObserveNotificationSeqMap.remove(token); return tokenToSessionInfoMap.remove(token); @@ -470,57 +449,6 @@ public class CoapTransportResource extends AbstractCoapTransportResource { } } - private class OtaPackageCallback implements TransportServiceCallback { - private final CoapExchange exchange; - - OtaPackageCallback(CoapExchange exchange) { - this.exchange = exchange; - } - - @Override - public void onSuccess(TransportProtos.GetOtaPackageResponseMsg msg) { - String title = exchange.getQueryParameter("title"); - String version = exchange.getQueryParameter("version"); - if (msg.getResponseStatus().equals(TransportProtos.ResponseStatus.SUCCESS)) { - String firmwareId = new UUID(msg.getOtaPackageIdMSB(), msg.getOtaPackageIdLSB()).toString(); - if (msg.getTitle().equals(title) && msg.getVersion().equals(version)) { - String strChunkSize = exchange.getQueryParameter("size"); - String strChunk = exchange.getQueryParameter("chunk"); - int chunkSize = StringUtils.isEmpty(strChunkSize) ? 0 : Integer.parseInt(strChunkSize); - int chunk = StringUtils.isEmpty(strChunk) ? 0 : Integer.parseInt(strChunk); - exchange.respond(CoAP.ResponseCode.CONTENT, transportContext.getOtaPackageDataCache().get(firmwareId, chunkSize, chunk)); - } - else if (firmwareId != null) { - sendOtaData(exchange, firmwareId); - } else { - exchange.respond(CoAP.ResponseCode.BAD_REQUEST); - } - } else { - exchange.respond(CoAP.ResponseCode.NOT_FOUND); - } - } - - @Override - public void onError(Throwable e) { - log.warn("Failed to process request", e); - exchange.respond(CoAP.ResponseCode.INTERNAL_SERVER_ERROR); - } - } - - private void sendOtaData(CoapExchange exchange, String firmwareId) { - Response response = new Response(CoAP.ResponseCode.CONTENT); - byte[] fwData = transportContext.getOtaPackageDataCache().get(firmwareId); - if (fwData != null && fwData.length > 0) { - response.setPayload(fwData); - if (exchange.getRequestOptions().getBlock2() != null) { - int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); - boolean moreFlag = fwData.length > chunkSize; - response.getOptions().setBlock2(chunkSize, moreFlag, 0); - } - exchange.respond(response); - } - } - private static class CoapSessionListener implements SessionMsgListener { private final CoapTransportResource coapTransportResource; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index ce1618fe99..72be5e6f1e 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.coapserver.CoapServerService; import org.thingsboard.server.coapserver.TbCoapServerComponent; +import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.transport.coap.efento.CoapEfentoTransportResource; import javax.annotation.PostConstruct; @@ -59,6 +60,8 @@ public class CoapTransportService implements TbTransportService { efento.add(efentoMeasurementsTransportResource); coapServer.add(api); coapServer.add(efento); + coapServer.add(new OtaPackageTransportResource(coapTransportContext, OtaPackageType.FIRMWARE)); + coapServer.add(new OtaPackageTransportResource(coapTransportContext, OtaPackageType.SOFTWARE)); log.info("CoAP transport started!"); } diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java new file mode 100644 index 0000000000..60e8f55352 --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.coap.CoAP; +import org.eclipse.californium.core.coap.Request; +import org.eclipse.californium.core.coap.Response; +import org.eclipse.californium.core.network.Exchange; +import org.eclipse.californium.core.server.resources.CoapExchange; +import org.eclipse.californium.core.server.resources.Resource; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.security.DeviceTokenCredentials; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Slf4j +public class OtaPackageTransportResource extends AbstractCoapTransportResource { + private static final int ACCESS_TOKEN_POSITION = 2; + + private final OtaPackageType otaPackageType; + + public OtaPackageTransportResource(CoapTransportContext ctx, OtaPackageType otaPackageType) { + super(ctx, otaPackageType.getKeyPrefix()); + this.otaPackageType = otaPackageType; + } + + @Override + protected void processHandleGet(CoapExchange exchange) { + log.trace("Processing {}", exchange.advanced().getRequest()); + exchange.accept(); + Exchange advanced = exchange.advanced(); + Request request = advanced.getRequest(); + processAccessTokenRequest(exchange, request); + } + + @Override + protected void processHandlePost(CoapExchange exchange) { + exchange.respond(CoAP.ResponseCode.METHOD_NOT_ALLOWED); + } + + private void processAccessTokenRequest(CoapExchange exchange, Request request) { + Optional credentials = decodeCredentials(request); + if (credentials.isEmpty()) { + exchange.respond(CoAP.ResponseCode.UNAUTHORIZED); + return; + } + transportService.process(DeviceTransportType.COAP, TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(credentials.get().getCredentialsId()).build(), + new CoapDeviceAuthCallback(transportContext, exchange, (sessionInfo, deviceProfile) -> { + getOtaPackageCallback(sessionInfo, exchange, otaPackageType); + })); + } + + private void getOtaPackageCallback(TransportProtos.SessionInfoProto sessionInfo, CoapExchange exchange, OtaPackageType firmwareType) { + TransportProtos.GetOtaPackageRequestMsg requestMsg = TransportProtos.GetOtaPackageRequestMsg.newBuilder() + .setTenantIdMSB(sessionInfo.getTenantIdMSB()) + .setTenantIdLSB(sessionInfo.getTenantIdLSB()) + .setDeviceIdMSB(sessionInfo.getDeviceIdMSB()) + .setDeviceIdLSB(sessionInfo.getDeviceIdLSB()) + .setType(firmwareType.name()).build(); + transportContext.getTransportService().process(sessionInfo, requestMsg, new OtaPackageCallback(exchange)); + } + + private Optional decodeCredentials(Request request) { + List uriPath = request.getOptions().getUriPath(); + if (uriPath.size() == ACCESS_TOKEN_POSITION) { + return Optional.of(new DeviceTokenCredentials(uriPath.get(ACCESS_TOKEN_POSITION - 1))); + } else { + return Optional.empty(); + } + } + + @Override + public Resource getChild(String name) { + return this; + } + + private class OtaPackageCallback implements TransportServiceCallback { + private final CoapExchange exchange; + + OtaPackageCallback(CoapExchange exchange) { + this.exchange = exchange; + } + + @Override + public void onSuccess(TransportProtos.GetOtaPackageResponseMsg msg) { + String title = exchange.getQueryParameter("title"); + String version = exchange.getQueryParameter("version"); + if (msg.getResponseStatus().equals(TransportProtos.ResponseStatus.SUCCESS)) { + String firmwareId = new UUID(msg.getOtaPackageIdMSB(), msg.getOtaPackageIdLSB()).toString(); + if ((title == null || msg.getTitle().equals(title)) && (version == null || msg.getVersion().equals(version))) { + String strChunkSize = exchange.getQueryParameter("size"); + String strChunk = exchange.getQueryParameter("chunk"); + int chunkSize = StringUtils.isEmpty(strChunkSize) ? 0 : Integer.parseInt(strChunkSize); + int chunk = StringUtils.isEmpty(strChunk) ? 0 : Integer.parseInt(strChunk); + respondOtaPackage(exchange, transportContext.getOtaPackageDataCache().get(firmwareId, chunkSize, chunk)); + } else { + exchange.respond(CoAP.ResponseCode.BAD_REQUEST); + } + } else { + exchange.respond(CoAP.ResponseCode.NOT_FOUND); + } + } + + @Override + public void onError(Throwable e) { + log.warn("Failed to process request", e); + exchange.respond(CoAP.ResponseCode.INTERNAL_SERVER_ERROR); + } + } + + private void respondOtaPackage(CoapExchange exchange, byte[] data) { + Response response = new Response(CoAP.ResponseCode.CONTENT); + if (data != null && data.length > 0) { + response.setPayload(data); + if (exchange.getRequestOptions().getBlock2() != null) { + int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); + boolean moreFlag = data.length > chunkSize; + response.getOptions().setBlock2(chunkSize, moreFlag, 0); + } + exchange.respond(response); + } + } + +} From 209c93064a2b6251ebf95accf08b4684d884b83b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Jun 2021 19:36:15 +0300 Subject: [PATCH 090/101] Fix issue on LWM2M Security Store --- .../coap/OtaPackageTransportResource.java | 4 +-- .../server/LwM2mTransportCoapResource.java | 6 ++--- .../server/LwM2mTransportServerHelper.java | 26 ++----------------- .../server/store/TbLwM2mSecurityStore.java | 3 ++- 4 files changed, 9 insertions(+), 30 deletions(-) diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java index 60e8f55352..cb403c5e82 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java @@ -134,8 +134,8 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { response.setPayload(data); if (exchange.getRequestOptions().getBlock2() != null) { int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); - boolean moreFlag = data.length > chunkSize; - response.getOptions().setBlock2(chunkSize, moreFlag, 0); + boolean lastFlag = data.length > chunkSize; + response.getOptions().setBlock2(chunkSize, lastFlag, 0); } exchange.respond(response); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java index 229ec1a08c..b996922ded 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java @@ -143,9 +143,9 @@ public class LwM2mTransportCoapResource extends AbstractLwM2mTransportResource { response.setPayload(fwData); if (exchange.getRequestOptions().getBlock2() != null) { int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); - boolean moreFlag = fwData.length > chunkSize; - response.getOptions().setBlock2(chunkSize, moreFlag, 0); - log.warn("92) with blokc2 Send currentId: [{}], length: [{}], chunkSize [{}], moreFlag [{}]", currentId.toString(), fwData.length, chunkSize, moreFlag); + boolean lastFlag = fwData.length > chunkSize; + response.getOptions().setBlock2(chunkSize, lastFlag, 0); + log.warn("92) with blokc2 Send currentId: [{}], length: [{}], chunkSize [{}], moreFlag [{}]", currentId.toString(), fwData.length, chunkSize, lastFlag); } else { log.warn("92) with block1 Send currentId: [{}], length: [{}], ", currentId.toString(), fwData.length); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java index 9e36d35076..a50e979b77 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java @@ -76,32 +76,11 @@ public class LwM2mTransportServerHelper { return TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) * 1000L + (atomicTs.getAndIncrement() % 1000); } - /** - * send to Thingsboard Attribute || Telemetry - * - * @param msg - JsonObject: [{name: value}] - * @return - dummyWriteReplace {\"targetIdVer\":\"/19_1.0/0/0\",\"value\":0082} - */ - private TransportServiceCallback getPubAckCallbackSendAttrTelemetry(final T msg) { - return new TransportServiceCallback<>() { - @Override - public void onSuccess(Void dummy) { - log.trace("Success to publish msg: {}, dummy: {}", msg, dummy); - } - - @Override - public void onError(Throwable e) { - log.trace("[{}] Failed to publish msg: {}", msg, e); - } - }; - } - public void sendParametersOnThingsboardAttribute(List result, SessionInfoProto sessionInfo) { PostAttributeMsg.Builder request = PostAttributeMsg.newBuilder(); request.addAllKv(result); PostAttributeMsg postAttributeMsg = request.build(); - TransportServiceCallback call = this.getPubAckCallbackSendAttrTelemetry(postAttributeMsg); - context.getTransportService().process(sessionInfo, postAttributeMsg, this.getPubAckCallbackSendAttrTelemetry(call)); + context.getTransportService().process(sessionInfo, postAttributeMsg, TransportServiceCallback.EMPTY); } public void sendParametersOnThingsboardTelemetry(List result, SessionInfoProto sessionInfo) { @@ -111,8 +90,7 @@ public class LwM2mTransportServerHelper { builder.addAllKv(result); request.addTsKvList(builder.build()); PostTelemetryMsg postTelemetryMsg = request.build(); - TransportServiceCallback call = this.getPubAckCallbackSendAttrTelemetry(postTelemetryMsg); - context.getTransportService().process(sessionInfo, postTelemetryMsg, this.getPubAckCallbackSendAttrTelemetry(call)); + context.getTransportService().process(sessionInfo, postTelemetryMsg, TransportServiceCallback.EMPTY); } /** diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java index cbdb9faf07..293e68ec39 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java @@ -85,6 +85,7 @@ public class TbLwM2mSecurityStore implements TbEditableSecurityStore { @Override public void remove(String endpoint) { - securityStore.remove(endpoint); + //TODO: Make sure we delay removal of security store from endpoint due to reg/unreg race condition. +// securityStore.remove(endpoint); } } From be3e97d3e161933b93fdc6affc300e7dae7d0a15 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 18 Jun 2021 12:24:31 +0300 Subject: [PATCH 091/101] Improvements for CoAP --- .../lwm2m/NoSecLwM2MIntegrationTest.java | 2 +- .../lwm2m/X509LwM2MIntegrationTest.java | 7 +++- .../coapserver/DefaultCoapServerService.java | 13 ++++++- .../coap/OtaPackageTransportResource.java | 37 +++++++++++++++++++ .../uplink/DefaultLwM2MUplinkMsgHandler.java | 1 + 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java index e94f432f64..d98a775c8c 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java @@ -53,7 +53,7 @@ public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { private final int PORT = 5685; private final Security SECURITY = noSec("coap://localhost:" + PORT, 123); private final NetworkConfig COAP_CONFIG = new NetworkConfig().setString("COAP_PORT", Integer.toString(PORT)); - private final String ENDPOINT = "deviceAEndpoint"; + private final String ENDPOINT = "noSecEndpoint"; private Device createDevice() throws Exception { Device device = new Device(); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java index a5efff968a..661f7c5474 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java @@ -19,6 +19,7 @@ import org.eclipse.californium.core.network.config.NetworkConfig; import org.eclipse.leshan.client.object.Security; import org.jetbrains.annotations.NotNull; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; @@ -74,11 +75,13 @@ public class X509LwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { return device; } + //TODO: use different endpoints to isolate tests. + @Ignore() @Test public void testConnectAndObserveTelemetry() throws Exception { createDeviceProfile(TRANSPORT_CONFIGURATION); X509ClientCredentials credentials = new X509ClientCredentials(); - credentials.setEndpoint(endpoint); + credentials.setEndpoint(endpoint+1); Device device = createDevice(credentials); SingleEntityFilter sef = new SingleEntityFilter(); @@ -96,7 +99,7 @@ public class X509LwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { wsClient.waitForReply(); wsClient.registerWaitForUpdate(); - LwM2MTestClient client = new LwM2MTestClient(executor, endpoint); + LwM2MTestClient client = new LwM2MTestClient(executor, endpoint+1); Security security = x509(serverUri, 123, clientX509Cert.getEncoded(), clientPrivateKeyFromCert.getEncoded(), serverX509Cert.getEncoded()); client.init(security, coapConfig); String msg = wsClient.waitForUpdate(); diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java index 93f4e76551..605952e456 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java @@ -36,6 +36,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.eclipse.californium.core.network.config.NetworkConfigDefaults.DEFAULT_BLOCKWISE_STATUS_LIFETIME; + @Slf4j @Component @TbCoapServerComponent @@ -91,7 +93,16 @@ public class DefaultCoapServerService implements CoapServerService { InetAddress addr = InetAddress.getByName(coapServerContext.getHost()); InetSocketAddress sockAddr = new InetSocketAddress(addr, coapServerContext.getPort()); noSecCoapEndpointBuilder.setInetSocketAddress(sockAddr); - noSecCoapEndpointBuilder.setNetworkConfig(NetworkConfig.getStandard()); + NetworkConfig networkConfig = new NetworkConfig(); + networkConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_STRICT_BLOCK2_OPTION, true); + networkConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_ENTITY_TOO_LARGE_AUTO_FAILOVER, true); + networkConfig.setLong(NetworkConfig.Keys.BLOCKWISE_STATUS_LIFETIME, DEFAULT_BLOCKWISE_STATUS_LIFETIME); + networkConfig.setInt(NetworkConfig.Keys.MAX_RESOURCE_BODY_SIZE, 256 * 1024 * 1024); + networkConfig.setString(NetworkConfig.Keys.RESPONSE_MATCHING, "RELAXED"); + networkConfig.setInt(NetworkConfig.Keys.PREFERRED_BLOCK_SIZE, 1024); + networkConfig.setInt(NetworkConfig.Keys.MAX_MESSAGE_SIZE, 1024); + networkConfig.setInt(NetworkConfig.Keys.MAX_RETRANSMIT, 4); + noSecCoapEndpointBuilder.setNetworkConfig(networkConfig); CoapEndpoint noSecCoapEndpoint = noSecCoapEndpointBuilder.build(); server.addEndpoint(noSecCoapEndpoint); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java index cb403c5e82..bc9fdd9282 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java @@ -20,8 +20,10 @@ import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.coap.Response; import org.eclipse.californium.core.network.Exchange; +import org.eclipse.californium.core.observe.ObserveRelation; import org.eclipse.californium.core.server.resources.CoapExchange; import org.eclipse.californium.core.server.resources.Resource; +import org.eclipse.californium.core.server.resources.ResourceObserver; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -41,6 +43,8 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { public OtaPackageTransportResource(CoapTransportContext ctx, OtaPackageType otaPackageType) { super(ctx, otaPackageType.getKeyPrefix()); + this.setObservable(true); + this.addObserver(new OtaPackageTransportResource.CoapResourceObserver()); this.otaPackageType = otaPackageType; } @@ -135,10 +139,43 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { if (exchange.getRequestOptions().getBlock2() != null) { int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); boolean lastFlag = data.length > chunkSize; + response.getOptions().setUriPath(exchange.getRequestOptions().getUriPathString()); response.getOptions().setBlock2(chunkSize, lastFlag, 0); } exchange.respond(response); } } + public class CoapResourceObserver implements ResourceObserver { + @Override + public void changedName(String old) { + + } + + @Override + public void changedPath(String old) { + + } + + @Override + public void addedChild(Resource child) { + + } + + @Override + public void removedChild(Resource child) { + + } + + @Override + public void addedObserveRelation(ObserveRelation relation) { + + } + + @Override + public void removedObserveRelation(ObserveRelation relation) { + + } + } + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 5b61ffc768..cd9c311b54 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -256,6 +256,7 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl LwM2mClient lwM2MClient = clientContext.getClientByEndpoint(registration.getEndpoint()); try { log.warn("[{}] [{{}] Client: update after Registration", registration.getEndpoint(), registration.getId()); + logService.log(lwM2MClient, String.format("[%s][%s] Updated registration.", registration.getId(), registration.getSocketAddress())); clientContext.updateRegistration(lwM2MClient, registration); TransportProtos.SessionInfoProto sessionInfo = lwM2MClient.getSession(); this.reportActivityAndRegister(sessionInfo); From fe3378eab125b6f59ae316890f3ed4384b16f114 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 18 Jun 2021 12:58:38 +0300 Subject: [PATCH 092/101] UI: Add SNMP device transport configuration --- .../SnmpDeviceTransportConfiguration.java | 9 +- ...-profile-communication-config.component.ts | 4 +- ...snmp-device-profile-mapping.component.scss | 3 + ...ofile-transport-configuration.component.ts | 2 +- .../device/data/device-data.component.ts | 36 +++++- ...evice-transport-configuration.component.ts | 44 +++++-- ...ice-transport-configuration.component.html | 121 +++++++++++++++++- ...evice-transport-configuration.component.ts | 112 +++++++++++++--- ui-ngx/src/app/shared/models/device.models.ts | 60 ++++++++- .../assets/locale/locale.constant-en_US.json | 25 +++- 10 files changed, 366 insertions(+), 50 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index a7bc143d81..d377ede985 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -51,6 +51,13 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur private String privacyPassphrase; private String engineId; + public SnmpDeviceTransportConfiguration() { + this.host = "localhost"; + this.port = 161; + this.protocolVersion = SnmpProtocolVersion.V2C; + this.community = "public"; + } + @Override public DeviceTransportType getType() { return DeviceTransportType.SNMP; @@ -76,7 +83,7 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur isValid = StringUtils.isNotBlank(username) && StringUtils.isNotBlank(securityName) && contextName != null && authenticationProtocol != null && StringUtils.isNotBlank(authenticationPassphrase) - && privacyProtocol != null && privacyPassphrase != null && engineId != null; + && privacyProtocol != null && StringUtils.isNotBlank(privacyPassphrase) && engineId != null; break; } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts index d683d46880..7af2140efe 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts @@ -179,7 +179,7 @@ export class SnmpDeviceProfileCommunicationConfigComponent implements OnInit, On if (isUndefinedOrNull(value)) { value = { spec: this.getFirstUnusedSeverity(), - queryingFrequencyMs: 0, + queryingFrequencyMs: 5000, mappings: null }; } @@ -196,7 +196,7 @@ export class SnmpDeviceProfileCommunicationConfigComponent implements OnInit, On ).subscribe(spec => { if (this.isShowFrequency(spec)) { form.addControl('queryingFrequencyMs', - this.fb.control(0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); + this.fb.control(5000, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); } else { form.removeControl('queryingFrequencyMs'); } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss index 94cf2290ac..5972aee722 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss @@ -40,6 +40,9 @@ .mat-form-field-underline { bottom: 0; } + .mat-form-field-subscript-wrapper{ + margin-top: 1.8em; + } } } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts index a384acbae3..400199ef9c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts @@ -86,7 +86,7 @@ export class SnmpDeviceProfileTransportConfigurationComponent implements OnInit, ngOnInit(): void { this.snmpDeviceProfileTransportConfigurationFormGroup = this.fb.group({ - timeoutMs: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + timeoutMs: [500, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], retries: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], communicationConfigs: [null, Validators.required], }); diff --git a/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts b/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts index 1a510692db..a175764d89 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts @@ -15,7 +15,16 @@ /// import { Component, forwardRef, Input, OnInit } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; import { Store } from '@ngrx/store'; import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; @@ -29,13 +38,20 @@ import { selector: 'tb-device-data', templateUrl: './device-data.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => DeviceDataComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => DeviceDataComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => DeviceDataComponent), + multi: true + }, + ] }) -export class DeviceDataComponent implements ControlValueAccessor, OnInit { +export class DeviceDataComponent implements ControlValueAccessor, OnInit, Validator { deviceDataFormGroup: FormGroup; @@ -97,6 +113,12 @@ export class DeviceDataComponent implements ControlValueAccessor, OnInit { this.deviceDataFormGroup.patchValue({transportConfiguration: value?.transportConfiguration}, {emitEvent: false}); } + validate(): ValidationErrors | null { + return this.deviceDataFormGroup.valid ? null : { + deviceDataForm: false + }; + } + private updateModel() { let deviceData: DeviceData = null; if (this.deviceDataFormGroup.valid) { diff --git a/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts index 0e9d7e24a9..82359232a9 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts @@ -15,27 +15,39 @@ /// import { Component, forwardRef, Input, OnInit } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; import { Store } from '@ngrx/store'; import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { - DeviceTransportConfiguration, - DeviceTransportType -} from '@shared/models/device.models'; +import { DeviceTransportConfiguration, DeviceTransportType } from '@shared/models/device.models'; import { deepClone } from '@core/utils'; @Component({ selector: 'tb-device-transport-configuration', templateUrl: './device-transport-configuration.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => DeviceTransportConfigurationComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => DeviceTransportConfigurationComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => DeviceTransportConfigurationComponent), + multi: true + }] }) -export class DeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit { +export class DeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit, Validator { deviceTransportType = DeviceTransportType; @@ -92,7 +104,15 @@ export class DeviceTransportConfigurationComponent implements ControlValueAccess if (configuration) { delete configuration.type; } - this.deviceTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + setTimeout(() => { + this.deviceTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + }, 0); + } + + validate(): ValidationErrors | null { + return this.deviceTransportConfigurationFormGroup.valid ? null : { + deviceTransportConfiguration: false + }; } private updateModel() { diff --git a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html index fc9f615db9..dadc98425e 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html @@ -15,10 +15,119 @@ limitations under the License. --> -

- - + +
+ + device-profile.snmp.host + + + {{ 'device-profile.snmp.host-required' | translate }} + + + + device-profile.snmp.port + + + {{ 'device-profile.snmp.port-required' | translate }} + + + {{ 'device-profile.snmp.port-format' | translate }} + + +
+ + device-profile.snmp.protocol-version + + + {{ snmpDeviceProtocolVersion | lowercase }} + + + + {{ 'device-profile.snmp.protocol-version-required' | translate }} + + +
+ + device-profile.snmp.community + + + {{ 'device-profile.snmp.community-required' | translate }} + + +
+
+
+ + device-profile.snmp.user-name + + + {{ 'device-profile.snmp.user-name-required' | translate }} + + + + device-profile.snmp.security-name + + + {{ 'device-profile.snmp.security-name-required' | translate }} + + +
+
+ + device-profile.snmp.authentication-protocol + + + {{ snmpAuthenticationProtocolTranslation.get(snmpAuthenticationProtocol) }} + + + + {{ 'device-profile.snmp.authentication-protocol-required' | translate }} + + + + device-profile.snmp.authentication-passphrase + + + {{ 'device-profile.snmp.authentication-passphrase-required' | translate }} + + +
+
+ + device-profile.snmp.privacy-protocol + + + {{ snmpPrivacyProtocolTranslation.get(snmpPrivacyProtocol) }} + + + + {{ 'device-profile.snmp.privacy-protocol-required' | translate }} + + + + device-profile.snmp.privacy-passphrase + + + {{ 'device-profile.snmp.privacy-passphrase-required' | translate }} + + +
+
+ + device-profile.snmp.context-name + + + + device-profile.snmp.engine-id + + +
+
diff --git a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts index 165a2906f4..f35d538bfa 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts @@ -14,31 +14,57 @@ /// limitations under the License. /// -import {Component, forwardRef, Input, OnInit} from '@angular/core'; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; -import {Store} from '@ngrx/store'; -import {AppState} from '@app/core/core.state'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { DeviceTransportConfiguration, DeviceTransportType, - SnmpDeviceTransportConfiguration + SnmpAuthenticationProtocol, + SnmpAuthenticationProtocolTranslationMap, + SnmpDeviceProtocolVersion, + SnmpDeviceTransportConfiguration, + SnmpPrivacyProtocol, + SnmpPrivacyProtocolTranslationMap } from '@shared/models/device.models'; +import { isDefinedAndNotNull } from '@core/utils'; @Component({ selector: 'tb-snmp-device-transport-configuration', templateUrl: './snmp-device-transport-configuration.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => SnmpDeviceTransportConfigurationComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceTransportConfigurationComponent), + multi: true + }, { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceTransportConfigurationComponent), + multi: true + }] }) -export class SnmpDeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit { +export class SnmpDeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit, Validator { snmpDeviceTransportConfigurationFormGroup: FormGroup; + snmpDeviceProtocolVersions = Object.values(SnmpDeviceProtocolVersion); + snmpAuthenticationProtocols = Object.values(SnmpAuthenticationProtocol); + snmpAuthenticationProtocolTranslation = SnmpAuthenticationProtocolTranslationMap; + snmpPrivacyProtocols = Object.values(SnmpPrivacyProtocol); + snmpPrivacyProtocolTranslation = SnmpPrivacyProtocolTranslationMap; + private requiredValue: boolean; get required(): boolean { @@ -53,8 +79,7 @@ export class SnmpDeviceTransportConfigurationComponent implements ControlValueAc @Input() disabled: boolean; - private propagateChange = (v: any) => { - }; + private propagateChange = (v: any) => { }; constructor(private store: Store, private fb: FormBuilder) { @@ -69,13 +94,33 @@ export class SnmpDeviceTransportConfigurationComponent implements ControlValueAc ngOnInit() { this.snmpDeviceTransportConfigurationFormGroup = this.fb.group({ - configuration: [null, Validators.required] + host: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + port: [null, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + protocolVersion: [SnmpDeviceProtocolVersion.V2C, Validators.required], + community: ['public', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + username: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + securityName: ['public', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + contextName: [null], + authenticationProtocol: [SnmpAuthenticationProtocol.SHA_512, Validators.required], + authenticationPassphrase: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + privacyProtocol: [SnmpPrivacyProtocol.DES, Validators.required], + privacyPassphrase: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + engineId: [''] + }); + this.snmpDeviceTransportConfigurationFormGroup.get('protocolVersion').valueChanges.subscribe((protocol: SnmpDeviceProtocolVersion) => { + this.updateDisabledFormValue(protocol); }); this.snmpDeviceTransportConfigurationFormGroup.valueChanges.subscribe(() => { this.updateModel(); }); } + validate(): ValidationErrors | null { + return this.snmpDeviceTransportConfigurationFormGroup.valid ? null : { + snmpDeviceTransportConfiguration: false + }; + } + setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; if (this.disabled) { @@ -86,13 +131,46 @@ export class SnmpDeviceTransportConfigurationComponent implements ControlValueAc } writeValue(value: SnmpDeviceTransportConfiguration | null): void { - this.snmpDeviceTransportConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + if (isDefinedAndNotNull(value)) { + this.snmpDeviceTransportConfigurationFormGroup.patchValue(value, {emitEvent: false}); + if (this.snmpDeviceTransportConfigurationFormGroup.enabled) { + this.updateDisabledFormValue(value.protocolVersion || SnmpDeviceProtocolVersion.V2C); + } + } + } + + isV3protocolVersion(): boolean { + return this.snmpDeviceTransportConfigurationFormGroup.get('protocolVersion').value === SnmpDeviceProtocolVersion.V3; + } + + private updateDisabledFormValue(protocol: SnmpDeviceProtocolVersion) { + if (protocol === SnmpDeviceProtocolVersion.V3) { + this.snmpDeviceTransportConfigurationFormGroup.get('community').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('username').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('securityName').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('contextName').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationProtocol').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationPassphrase').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyProtocol').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyPassphrase').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('engineId').enable({emitEvent: false}); + } else { + this.snmpDeviceTransportConfigurationFormGroup.get('community').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('username').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('securityName').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('contextName').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationProtocol').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationPassphrase').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyProtocol').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyPassphrase').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('engineId').disable({emitEvent: false}); + } } private updateModel() { let configuration: DeviceTransportConfiguration = null; if (this.snmpDeviceTransportConfigurationFormGroup.valid) { - configuration = this.snmpDeviceTransportConfigurationFormGroup.getRawValue().configuration; + configuration = this.snmpDeviceTransportConfigurationFormGroup.value; configuration.type = DeviceTransportType.SNMP; } this.propagateChange(configuration); diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 568faeb452..a766c3a6c3 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -362,7 +362,7 @@ export function createDeviceProfileTransportConfiguration(type: DeviceTransportT break; case DeviceTransportType.SNMP: const snmpTransportConfiguration: SnmpDeviceProfileTransportConfiguration = { - timeoutMs: 0, + timeoutMs: 500, retries: 0, communicationConfigs: null }; @@ -394,7 +394,12 @@ export function createDeviceTransportConfiguration(type: DeviceTransportType): D transportConfiguration = {...lwm2mTransportConfiguration, type: DeviceTransportType.LWM2M}; break; case DeviceTransportType.SNMP: - const snmpTransportConfiguration: SnmpDeviceTransportConfiguration = {}; + const snmpTransportConfiguration: SnmpDeviceTransportConfiguration = { + host: 'localhost', + port: 161, + protocolVersion: SnmpDeviceProtocolVersion.V2C, + community: 'public' + }; transportConfiguration = {...snmpTransportConfiguration, type: DeviceTransportType.SNMP}; break; } @@ -572,8 +577,57 @@ export interface Lwm2mDeviceTransportConfiguration { [key: string]: any; } +export enum SnmpDeviceProtocolVersion { + V1 = 'V1', + V2C = 'V2C', + V3 = 'V3' +} + +export enum SnmpAuthenticationProtocol { + SHA_1 = 'SHA_1', + SHA_224 = 'SHA_224', + SHA_256 = 'SHA_256', + SHA_384 = 'SHA_384', + SHA_512 = 'SHA_512', + MD5 = 'MD%' +} + +export const SnmpAuthenticationProtocolTranslationMap = new Map([ + [SnmpAuthenticationProtocol.SHA_1, 'SHA-1'], + [SnmpAuthenticationProtocol.SHA_224, 'SHA-224'], + [SnmpAuthenticationProtocol.SHA_256, 'SHA-256'], + [SnmpAuthenticationProtocol.SHA_384, 'SHA-384'], + [SnmpAuthenticationProtocol.SHA_512, 'SHA-512'], + [SnmpAuthenticationProtocol.MD5, 'MD5'] +]); + +export enum SnmpPrivacyProtocol { + DES = 'DES', + AES_128 = 'AES_128', + AES_192 = 'AES_192', + AES_256 = 'AES_256' +} + +export const SnmpPrivacyProtocolTranslationMap = new Map([ + [SnmpPrivacyProtocol.DES, 'DES'], + [SnmpPrivacyProtocol.AES_128, 'AES-128'], + [SnmpPrivacyProtocol.AES_192, 'AES-192'], + [SnmpPrivacyProtocol.AES_256, 'AES-256'], +]); + export interface SnmpDeviceTransportConfiguration { - [key: string]: any; + host?: string; + port?: number; + protocolVersion?: SnmpDeviceProtocolVersion; + community?: string; + username?: string; + securityName?: string; + contextName?: string; + authenticationProtocol?: SnmpAuthenticationProtocol; + authenticationPassphrase?: string; + privacyProtocol?: SnmpPrivacyProtocol; + privacyPassphrase?: string; + engineId?: string; } export type DeviceTransportConfigurations = DefaultDeviceTransportConfiguration & diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 95dc0c2d72..ff07be44da 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1301,16 +1301,35 @@ "snmp": { "add-communication-config": "Add communication config", "add-mapping": "Add mapping", + "authentication-passphrase": "Authentication passphrase", + "authentication-passphrase-required": "Authentication passphrase is required.", + "authentication-protocol": "Authentication protocol", + "authentication-protocol-required": "Authentication protocol is required.", "communication-configs": "Communication configs", + "community": "Community string", + "community-required": "Community string is required.", + "context-name": "Context name", "data-key": "Data key", "data-key-required": "Data key is required.", "data-type": "Data type", "data-type-required": "Data type is required.", + "engine-id": "Engine ID", + "host": "Host", + "host-required": "Host is required.", "oid": "OID", "oid-pattern": "Invalid OID format.", "oid-required": "OID is required.", "please-add-communication-config": "Please add communication config", "please-add-mapping-config": "Please add mapping config", + "port": "Port", + "port-format": "Invalid port format.", + "port-required": "Port is required.", + "privacy-passphrase": "Privacy passphrase", + "privacy-passphrase-required": "Privacy passphrase is required.", + "privacy-protocol": "Privacy protocol", + "privacy-protocol-required": "Privacy protocol is required.", + "protocol-version": "Protocol version", + "protocol-version-required": "Protocol version is required.", "querying-frequency": "Querying frequency, ms", "querying-frequency-invalid-format": "Querying frequency must be a positive integer.", "querying-frequency-required": "Querying frequency is required.", @@ -1319,9 +1338,13 @@ "retries-required": "Retries is required.", "scope": "Scope", "scope-required": "Scope is required.", + "security-name": "Security name", + "security-name-required": "Security name is required.", "timeout-ms": "Timeout, ms", "timeout-ms-invalid-format": "Timeout must be a positive integer.", - "timeout-ms-required": "Timeout is required." + "timeout-ms-required": "Timeout is required.", + "user-name": "User name", + "user-name-required": "User name is required." } }, "dialog": { From e3c523cc6454db8a6b10aa63b1c542444028a45e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 18 Jun 2021 14:33:31 +0300 Subject: [PATCH 093/101] Write replace command fix --- .../lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java | 2 +- .../server/transport/lwm2m/server/rpc/IdOrKeyRequest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java index 5de223fa2f..0432a5b742 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java @@ -206,7 +206,7 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { } private void sendWriteReplaceRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { - RpcWriteUpdateRequest requestBody = JacksonUtil.fromString(requestMsg.getParams(), RpcWriteUpdateRequest.class); + RpcWriteReplaceRequest requestBody = JacksonUtil.fromString(requestMsg.getParams(), RpcWriteReplaceRequest.class); TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId) .value(requestBody.getValue()) .timeout(this.config.getTimeout()).build(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java index 1055b01232..77d8cd3ea1 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.transport.lwm2m.server.rpc; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; @Data +@JsonIgnoreProperties(ignoreUnknown = true) public class IdOrKeyRequest { private String key; From e470a032a9f2fa12d949f6ed83df3ba7ef39e58b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 18 Jun 2021 17:21:35 +0300 Subject: [PATCH 094/101] improved countByDeviceProfileAndEmptyOtaPackage api --- .../server/controller/DeviceController.java | 10 ++++++---- .../java/org/thingsboard/rest/client/RestClient.java | 2 +- ui-ngx/src/app/core/http/ota-package.service.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index b13393515c..5c7df50d09 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -782,15 +782,17 @@ public class DeviceController extends BaseController { } @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/devices/count/{otaPackageType}", method = RequestMethod.GET) + @RequestMapping(value = "/devices/count/{otaPackageType}/{deviceProfileId}", method = RequestMethod.GET) @ResponseBody - public Long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(@PathVariable("otaPackageType") String otaPackageType, - @RequestParam String deviceProfileId) throws ThingsboardException { + public Long countByDeviceProfileAndEmptyOtaPackage(@PathVariable("otaPackageType") String otaPackageType, + @PathVariable("deviceProfileId") String deviceProfileId) throws ThingsboardException { checkParameter("OtaPackageType", otaPackageType); checkParameter("DeviceProfileId", deviceProfileId); try { return deviceService.countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage( - getCurrentUser().getTenantId(), new DeviceProfileId(UUID.fromString(deviceProfileId)), OtaPackageType.valueOf(otaPackageType)); + getTenantId(), + new DeviceProfileId(UUID.fromString(deviceProfileId)), + OtaPackageType.valueOf(otaPackageType)); } catch (Exception e) { throw handleException(e); } diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index b716935b2a..6fbf28f24e 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -1257,7 +1257,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { params.put("deviceProfileId", deviceProfileId.getId().toString()); return restTemplate.exchange( - baseURL + "/api/devices/count/{otaPackageType}?deviceProfileId={deviceProfileId}", + baseURL + "/api/devices/count/{otaPackageType}/{deviceProfileId}", HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference() { diff --git a/ui-ngx/src/app/core/http/ota-package.service.ts b/ui-ngx/src/app/core/http/ota-package.service.ts index 1f9a36882a..a806751825 100644 --- a/ui-ngx/src/app/core/http/ota-package.service.ts +++ b/ui-ngx/src/app/core/http/ota-package.service.ts @@ -133,7 +133,7 @@ export class OtaPackageService { } public countUpdateDeviceAfterChangePackage(type: OtaUpdateType, entityId: EntityId, config?: RequestConfig): Observable { - return this.http.get(`/api/devices/count/${type}?deviceProfileId=${entityId.id}`, defaultHttpOptionsFromConfig(config)); + return this.http.get(`/api/devices/count/${type}/${entityId.id}`, defaultHttpOptionsFromConfig(config)); } public confirmDialogUpdatePackage(entity: BaseData&OtaPagesIds, From 5b998717e79f7fcde249b9a0a3686a2c0f5be68e Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 18 Jun 2021 18:00:12 +0300 Subject: [PATCH 095/101] Remove invalid test --- .../CachedAttributesServiceTest.java | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java diff --git a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java deleted file mode 100644 index af980a6301..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL - * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. - * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. - * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. - */ -package org.thingsboard.server.dao.attributes; - -import com.google.common.util.concurrent.MoreExecutors; -import org.junit.Test; -import org.thingsboard.server.dao.cache.CacheConfiguration; -import org.thingsboard.server.dao.cache.CacheExecutorService; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.willCallRealMethod; -import static org.mockito.Mockito.mock; - -public class CachedAttributesServiceTest { - - public static final String REDIS = "redis"; - - @Test - public void givenLocalCacheTypeName_whenEquals_thenOK() { - assertThat(CachedAttributesService.LOCAL_CACHE_TYPE, is("caffeine")); - } - - @Test - public void givenCacheType_whenGetExecutor_thenDirectExecutor() { - CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); - CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); - willCallRealMethod().given(cachedAttributesService).getExecutor(any(), any()); - - assertThat(cachedAttributesService.getExecutor(null, cacheExecutorService), is(MoreExecutors.directExecutor())); - - CacheConfiguration cacheConfiguration = new CacheConfiguration(); - cacheConfiguration.setType(null); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); - - cacheConfiguration.setType(""); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); - - cacheConfiguration.setType(CachedAttributesService.LOCAL_CACHE_TYPE); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); - - } - - @Test - public void givenCacheType_whenGetExecutor_thenReturnCacheExecutorService() { - CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); - CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); - willCallRealMethod().given(cachedAttributesService).getExecutor(any(CacheConfiguration.class), any(CacheExecutorService.class)); - - CacheConfiguration cacheConfiguration = new CacheConfiguration(); - cacheConfiguration.setType(REDIS); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); - - cacheConfiguration.setType("unknownCacheType"); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); - - } - -} \ No newline at end of file From d64a0c382d7bac67224db60879ce640e8d3bd59a Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 18 Jun 2021 18:22:48 +0300 Subject: [PATCH 096/101] Fix test properties --- dao/src/test/resources/application-test.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index d7a960471b..a257c8fbce 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -10,6 +10,7 @@ audit-log.default_query_period=30 audit-log.sink.type=none cache.type=caffeine +cache.maximumPoolSize=16 #cache.type=redis caffeine.specs.relations.timeToLiveInMinutes=1440 From a7a7c801f2ea59f72cec1c92007fb9c0ad6d4804 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 18 Jun 2021 18:31:52 +0300 Subject: [PATCH 097/101] Fix license header --- .../dao/cache/CacheExecutorService.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java index ca00e41d79..9f0f54bb46 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.server.dao.cache; From c03893fb570ae659e41493f4801d62463207d956 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Sat, 19 Jun 2021 16:58:32 +0300 Subject: [PATCH 098/101] LWM2M: fix bug update FwUrl in client after change in profile --- .../coapserver/DefaultCoapServerService.java | 18 +++++++++++- .../transport/coap/CoapTransportService.java | 13 ++++++++- .../coap/OtaPackageTransportResource.java | 15 ++++++++-- ...LwM2mCredentialsSecurityInfoValidator.java | 3 ++ .../server/LwM2mTransportCoapResource.java | 4 +-- .../ota/DefaultLwM2MOtaUpdateService.java | 29 +++++++++++++++++-- .../lwm2m/server/ota/LwM2MClientOtaInfo.java | 8 +++-- .../server/ota/LwM2MOtaUpdateService.java | 4 +++ .../uplink/DefaultLwM2MUplinkMsgHandler.java | 27 +++++++++++++++-- ...ile-transport-configuration.component.html | 4 +-- ...ofile-transport-configuration.component.ts | 6 ++-- .../lwm2m/lwm2m-profile-config.models.ts | 4 +-- 12 files changed, 112 insertions(+), 23 deletions(-) diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java index 93f4e76551..211ccecc3e 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java @@ -36,6 +36,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.eclipse.californium.core.network.config.NetworkConfigDefaults.DEFAULT_BLOCKWISE_STATUS_LIFETIME; + @Slf4j @Component @TbCoapServerComponent @@ -91,7 +93,21 @@ public class DefaultCoapServerService implements CoapServerService { InetAddress addr = InetAddress.getByName(coapServerContext.getHost()); InetSocketAddress sockAddr = new InetSocketAddress(addr, coapServerContext.getPort()); noSecCoapEndpointBuilder.setInetSocketAddress(sockAddr); - noSecCoapEndpointBuilder.setNetworkConfig(NetworkConfig.getStandard()); + + NetworkConfig coapConfig = new NetworkConfig(); + coapConfig.setInt(NetworkConfig.Keys.COAP_PORT, 5683); + coapConfig.setInt(NetworkConfig.Keys.COAP_SECURE_PORT, 5684); + coapConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_STRICT_BLOCK2_OPTION, true); + coapConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_ENTITY_TOO_LARGE_AUTO_FAILOVER, true); + coapConfig.setLong(NetworkConfig.Keys.BLOCKWISE_STATUS_LIFETIME, DEFAULT_BLOCKWISE_STATUS_LIFETIME); + coapConfig.setInt(NetworkConfig.Keys.MAX_RESOURCE_BODY_SIZE, 256 * 1024 * 1024); + coapConfig.setString(NetworkConfig.Keys.RESPONSE_MATCHING, "RELAXED"); + coapConfig.setInt(NetworkConfig.Keys.PREFERRED_BLOCK_SIZE, 1024); + coapConfig.setInt(NetworkConfig.Keys.MAX_MESSAGE_SIZE, 1024); + + coapConfig.setInt(NetworkConfig.Keys.MAX_RETRANSMIT, 10); + + noSecCoapEndpointBuilder.setNetworkConfig(coapConfig); CoapEndpoint noSecCoapEndpoint = noSecCoapEndpointBuilder.build(); server.addEndpoint(noSecCoapEndpoint); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index 72be5e6f1e..a5adc36d7e 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -18,11 +18,12 @@ package org.thingsboard.server.transport.coap; import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; +import org.eclipse.californium.core.network.config.NetworkConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.coapserver.CoapServerService; import org.thingsboard.server.coapserver.TbCoapServerComponent; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.transport.coap.efento.CoapEfentoTransportResource; @@ -30,6 +31,8 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.net.UnknownHostException; +import static org.eclipse.californium.core.network.config.NetworkConfigDefaults.DEFAULT_BLOCKWISE_STATUS_LIFETIME; + @Service("CoapTransportService") @TbCoapServerComponent @Slf4j @@ -52,6 +55,14 @@ public class CoapTransportService implements TbTransportService { public void init() throws UnknownHostException { log.info("Starting CoAP transport..."); coapServer = coapServerService.getCoapServer(); + coapServer.getConfig().setBoolean(NetworkConfig.Keys.BLOCKWISE_STRICT_BLOCK2_OPTION, true); + coapServer.getConfig().setBoolean(NetworkConfig.Keys.BLOCKWISE_ENTITY_TOO_LARGE_AUTO_FAILOVER, true); + coapServer.getConfig().setLong(NetworkConfig.Keys.BLOCKWISE_STATUS_LIFETIME, DEFAULT_BLOCKWISE_STATUS_LIFETIME); + coapServer.getConfig().setInt(NetworkConfig.Keys.MAX_RESOURCE_BODY_SIZE, 256 * 1024 * 1024); + coapServer.getConfig().setString(NetworkConfig.Keys.RESPONSE_MATCHING, "RELAXED"); + coapServer.getConfig().setInt(NetworkConfig.Keys.PREFERRED_BLOCK_SIZE, 1024); + coapServer.getConfig().setInt(NetworkConfig.Keys.MAX_MESSAGE_SIZE, 1024); + coapServer.getConfig().setInt(NetworkConfig.Keys.MAX_RETRANSMIT, 10); CoapResource api = new CoapResource(API); api.add(new CoapTransportResource(coapTransportContext, coapServerService, V1)); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java index cb403c5e82..989239549c 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java @@ -17,11 +17,13 @@ package org.thingsboard.server.transport.coap; import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.coap.CoAP; +import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.coap.Response; import org.eclipse.californium.core.network.Exchange; import org.eclipse.californium.core.server.resources.CoapExchange; import org.eclipse.californium.core.server.resources.Resource; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -32,16 +34,21 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ExecutorService; @Slf4j public class OtaPackageTransportResource extends AbstractCoapTransportResource { private static final int ACCESS_TOKEN_POSITION = 2; private final OtaPackageType otaPackageType; + private final ExecutorService sendOtaDataOutUriLarge; public OtaPackageTransportResource(CoapTransportContext ctx, OtaPackageType otaPackageType) { super(ctx, otaPackageType.getKeyPrefix()); this.otaPackageType = otaPackageType; + + this.setObservable(true); + this.sendOtaDataOutUriLarge = ThingsBoardExecutors.newWorkStealingPool(10, "LwM2M sendOtaDataOutUriLarge"); } @Override @@ -132,11 +139,13 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { Response response = new Response(CoAP.ResponseCode.CONTENT); if (data != null && data.length > 0) { response.setPayload(data); + response.getOptions().setAccept(MediaTypeRegistry.APPLICATION_OCTET_STREAM); if (exchange.getRequestOptions().getBlock2() != null) { int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); - boolean lastFlag = data.length > chunkSize; - response.getOptions().setBlock2(chunkSize, lastFlag, 0); - } + boolean lastFlag = data.length <= chunkSize; + this.sendOtaDataOutUriLarge.submit(() -> { + response.getOptions().setBlock2(chunkSize, lastFlag, 0); + }); } exchange.respond(response); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java index 13d6fd6568..1ec48b31f5 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java @@ -58,10 +58,12 @@ public class LwM2mCredentialsSecurityInfoValidator { public TbLwM2MSecurityInfo getEndpointSecurityInfoByCredentialsId(String credentialsId, LwM2mTransportUtil.LwM2mTypeServer keyValue) { CountDownLatch latch = new CountDownLatch(1); final TbLwM2MSecurityInfo[] resultSecurityStore = new TbLwM2MSecurityInfo[1]; + log.warn("001) [{}]", credentialsId); context.getTransportService().process(ValidateDeviceLwM2MCredentialsRequestMsg.newBuilder().setCredentialsId(credentialsId).build(), new TransportServiceCallback<>() { @Override public void onSuccess(ValidateDeviceCredentialsResponse msg) { + log.warn("002) [{}] [{}]", credentialsId, msg); String credentialsBody = msg.getCredentials(); resultSecurityStore[0] = createSecurityInfo(credentialsId, credentialsBody, keyValue); resultSecurityStore[0].setMsg(msg); @@ -71,6 +73,7 @@ public class LwM2mCredentialsSecurityInfoValidator { @Override public void onError(Throwable e) { + log.warn("003) [{}] [{}] Failed to process credentials ", credentialsId, e); log.trace("[{}] [{}] Failed to process credentials ", credentialsId, e); resultSecurityStore[0] = createSecurityInfo(credentialsId, null, null); latch.countDown(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java index b996922ded..f6456a394c 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java @@ -25,8 +25,6 @@ import org.eclipse.californium.core.server.resources.CoapExchange; import org.eclipse.californium.core.server.resources.Resource; import org.eclipse.californium.core.server.resources.ResourceObserver; import org.thingsboard.server.cache.ota.OtaPackageDataCache; -import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; -import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -143,7 +141,7 @@ public class LwM2mTransportCoapResource extends AbstractLwM2mTransportResource { response.setPayload(fwData); if (exchange.getRequestOptions().getBlock2() != null) { int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); - boolean lastFlag = fwData.length > chunkSize; + boolean lastFlag = fwData.length <= chunkSize; response.getOptions().setBlock2(chunkSize, lastFlag, 0); log.warn("92) with blokc2 Send currentId: [{}], length: [{}], chunkSize [{}], moreFlag [{}]", currentId.toString(), fwData.length, chunkSize, lastFlag); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java index d44770e591..4126541176 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; +import org.thingsboard.server.transport.lwm2m.server.LwM2MSoftwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; @@ -174,6 +175,24 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl fwInfo.setCurrentName(name); } + @Override + public void onCurrentFirmwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl) { + log.debug("[{}] Current fw strategy: {}", client.getEndpoint(), newStrategy); + LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); + fwInfo.setFwStrategy(LwM2MFirmwareUpdateStrategy.fromStrategyFwByCode(newStrategy)); + fwInfo.setBaseUrl(newBaseUrl); + startFirmwareUpdateIfNeeded(client, fwInfo); + } + + @Override + public void onCurrentSoftwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl) { + log.debug("[{}] Current sw strategy: {}", client.getEndpoint(), newStrategy); + LwM2MClientOtaInfo swInfo = getOrInitSwInfo(client); + swInfo.setSwStrategy(LwM2MSoftwareUpdateStrategy.fromStrategySwByCode(newStrategy)); + swInfo.setBaseUrl(newBaseUrl); + startSoftwareUpdateIfNeeded(client, swInfo); + } + @Override public void onCurrentFirmwareVersion3Update(LwM2mClient client, String version) { log.debug("[{}] Current fw version: {}", client.getEndpoint(), version); @@ -250,6 +269,10 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl } } + private void startSoftwareUpdateIfNeeded(LwM2mClient client, LwM2MClientOtaInfo swInfo) { + + } + private void startFirmwareUpdateUsingUrl(LwM2mClient client, String url) { String targetIdVer = convertObjectIdToVersionedId(FW_URL_ID, client.getRegistration()); TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(targetIdVer).value(url).timeout(config.getTimeout()).build(); @@ -277,7 +300,7 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl UUID otaPackageId = new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB()); LwM2MFirmwareUpdateStrategy strategy; if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == 2) { - strategy = fwInfo.getStrategy(); + strategy = fwInfo.getFwStrategy(); } else { strategy = fwInfo.getDeliveryMethod() == 0 ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; } @@ -328,9 +351,9 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl return Optional.empty(); } - private LwM2MClientOtaInfo getOrInitFwInfo(LwM2mClient client) { + public LwM2MClientOtaInfo getOrInitFwInfo(LwM2mClient client) { //TODO: fetch state from the cache or DB. - return fwStates.computeIfAbsent(client.getEndpoint(), endpoint -> { + return this.fwStates.computeIfAbsent(client.getEndpoint(), endpoint -> { var profile = clientContext.getProfile(client.getProfileId()); return new LwM2MClientOtaInfo(endpoint, OtaPackageType.FIRMWARE, profile.getClientLwM2mSettings().getFwUpdateStrategy(), profile.getClientLwM2mSettings().getFwUpdateRecourse()); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java index 43d4f4acf5..74bf8d1fac 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java @@ -19,8 +19,9 @@ import lombok.Data; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.LwM2MSoftwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; +import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; import java.util.Optional; @@ -44,7 +45,8 @@ public class LwM2MClientOtaInfo { private Integer deliveryMethod; //TODO: use value from device if applicable; - private LwM2MFirmwareUpdateStrategy strategy; + private LwM2MFirmwareUpdateStrategy fwStrategy; + private LwM2MSoftwareUpdateStrategy swStrategy; private UpdateStateFw updateState; private UpdateResultFw updateResult; @@ -54,7 +56,7 @@ public class LwM2MClientOtaInfo { public LwM2MClientOtaInfo(String endpoint, OtaPackageType type, Integer strategyCode, String baseUrl) { this.endpoint = endpoint; this.type = type; - this.strategy = LwM2MFirmwareUpdateStrategy.fromStrategyFwByCode(strategyCode); + this.fwStrategy = LwM2MFirmwareUpdateStrategy.fromStrategyFwByCode(strategyCode); this.baseUrl = baseUrl; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java index 9c7905a5e8..644099ee9d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java @@ -31,6 +31,10 @@ public interface LwM2MOtaUpdateService { void onCurrentFirmwareNameUpdate(LwM2mClient client, String name); + void onCurrentFirmwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl); + + void onCurrentSoftwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl); + void onCurrentFirmwareVersion3Update(LwM2mClient client, String version); void onCurrentFirmwareVersion5Update(LwM2mClient client, String version); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 5b61ffc768..0810d1a423 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -37,7 +37,6 @@ import org.eclipse.leshan.server.registration.Registration; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -53,6 +52,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; +import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; +import org.thingsboard.server.transport.lwm2m.server.LwM2MSoftwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.LwM2mOtaConvert; import org.thingsboard.server.transport.lwm2m.server.LwM2mQueuedRequest; import org.thingsboard.server.transport.lwm2m.server.LwM2mSessionMsgListener; @@ -100,7 +101,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -779,6 +779,29 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl clients.forEach(client -> sendCancelObserveRequest(targetId, client)); } } + + // # 7.1 + // update value in fwInfo + if (!newProfile.getClientLwM2mSettings().getFwUpdateStrategy().equals(oldProfile.getClientLwM2mSettings().getFwUpdateStrategy()) + || (LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == newProfile.getClientLwM2mSettings().getFwUpdateStrategy() && + !newProfile.getClientLwM2mSettings().getFwUpdateRecourse().equals(oldProfile.getClientLwM2mSettings().getFwUpdateRecourse()))) { + clients.forEach(lwM2MClient -> { + otaService.onCurrentFirmwareStrategyUpdate(lwM2MClient, + newProfile.getClientLwM2mSettings().getFwUpdateStrategy(), + newProfile.getClientLwM2mSettings().getFwUpdateRecourse()); + }); + } + + //# 7.2 // update value in swInfo + if (!newProfile.getClientLwM2mSettings().getSwUpdateStrategy().equals(oldProfile.getClientLwM2mSettings().getSwUpdateStrategy()) + || (LwM2MSoftwareUpdateStrategy.TEMP_URL.code == newProfile.getClientLwM2mSettings().getSwUpdateStrategy() && + !newProfile.getClientLwM2mSettings().getSwUpdateRecourse().equals(oldProfile.getClientLwM2mSettings().getSwUpdateRecourse()))) { + clients.forEach(lwM2MClient -> { + otaService.onCurrentSoftwareStrategyUpdate(lwM2MClient, + newProfile.getClientLwM2mSettings().getFwUpdateStrategy(), + newProfile.getClientLwM2mSettings().getFwUpdateRecourse()); + }); + } } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.html index 30b78393d3..5286161651 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.html @@ -160,9 +160,9 @@ - + - + diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts index faf94eccce..f99f1629ab 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts @@ -97,7 +97,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro binding: [], bootstrapServer: [null, Validators.required], lwm2mServer: [null, Validators.required], - clientStrategy: [1, []], + clientOnlyObserveAfterConnect: [1, []], fwUpdateStrategy: [1, []], swUpdateStrategy: [1, []], fwUpdateRecourse: [{value: '', disabled: true}, []], @@ -216,7 +216,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro binding: this.configurationValue.bootstrap.servers.binding, bootstrapServer: this.configurationValue.bootstrap.bootstrapServer, lwm2mServer: this.configurationValue.bootstrap.lwm2mServer, - clientStrategy: this.configurationValue.clientLwM2mSettings.clientStrategy, + clientOnlyObserveAfterConnect: this.configurationValue.clientLwM2mSettings.clientOnlyObserveAfterConnect, fwUpdateStrategy: this.configurationValue.clientLwM2mSettings.fwUpdateStrategy || 1, swUpdateStrategy: this.configurationValue.clientLwM2mSettings.swUpdateStrategy || 1, fwUpdateRecourse: fwResource, @@ -257,7 +257,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro bootstrapServers.defaultMinPeriod = config.defaultMinPeriod; bootstrapServers.notifIfDisabled = config.notifIfDisabled; bootstrapServers.binding = config.binding; - this.configurationValue.clientLwM2mSettings.clientStrategy = config.clientStrategy; + this.configurationValue.clientLwM2mSettings.clientOnlyObserveAfterConnect = config.clientOnlyObserveAfterConnect; this.configurationValue.clientLwM2mSettings.fwUpdateStrategy = config.fwUpdateStrategy; this.configurationValue.clientLwM2mSettings.swUpdateStrategy = config.swUpdateStrategy; this.configurationValue.clientLwM2mSettings.fwUpdateRecourse = config.fwUpdateRecourse; diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts index 1f2d125d9f..594147e5af 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts @@ -168,7 +168,7 @@ export interface Lwm2mProfileConfigModels { } export interface ClientLwM2mSettings { - clientStrategy: string; + clientOnlyObserveAfterConnect: number; fwUpdateStrategy: number; swUpdateStrategy: number; fwUpdateRecourse: string; @@ -240,7 +240,7 @@ export function getDefaultProfileConfig(hostname?: any): Lwm2mProfileConfigModel function getDefaultProfileClientLwM2mSettingsConfig(): ClientLwM2mSettings { return { - clientStrategy: '1', + clientOnlyObserveAfterConnect: 1, fwUpdateStrategy: 1, swUpdateStrategy: 1, fwUpdateRecourse: DEFAULT_FW_UPDATE_RESOURCE, From a34ffe8a409e353cf73faeaf83e6cc22c518a0ed Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Sun, 20 Jun 2021 14:09:20 +0300 Subject: [PATCH 099/101] LWM2M: merge with master --- .../secure/LwM2MBootstrapSecurityStore.java | 8 +- ...LwM2mCredentialsSecurityInfoValidator.java | 9 +- .../TbLwM2MDtlsCertificateVerifier.java | 5 +- .../server/DefaultLwM2mTransportService.java | 2 +- .../server/LwM2mTransportCoapResource.java | 4 +- .../lwm2m/server/LwM2mTransportUtil.java | 352 ++---------------- .../lwm2m/server/client/LwM2mClient.java | 6 +- .../ota/DefaultLwM2MOtaUpdateService.java | 123 +++++- .../lwm2m/server/ota/LwM2MClientOtaInfo.java | 8 +- .../LwM2MFirmwareUpdateStrategy.java | 2 +- .../server/ota/firmware/UpdateDeliveryFw.java | 55 +++ .../{ => ota/firmware}/UpdateResultFw.java | 2 +- .../{ => ota/firmware}/UpdateStateFw.java | 2 +- .../LwM2MSoftwareUpdateStrategy.java | 2 +- .../server/ota/software/UpdateResultSw.java | 84 +++++ .../server/ota/software/UpdateStateSw.java | 62 +++ .../server/store/TbLwM2mSecurityStore.java | 11 +- .../uplink/DefaultLwM2MUplinkMsgHandler.java | 16 +- .../lwm2m/server/uplink/LwM2mTypeServer.java | 39 ++ 19 files changed, 408 insertions(+), 384 deletions(-) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/{ => ota/firmware}/LwM2MFirmwareUpdateStrategy.java (96%) create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateDeliveryFw.java rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/{ => ota/firmware}/UpdateResultFw.java (97%) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/{ => ota/firmware}/UpdateStateFw.java (96%) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/{ => ota/software}/LwM2MSoftwareUpdateStrategy.java (95%) create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateResultSw.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateStateSw.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mTypeServer.java diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapSecurityStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapSecurityStore.java index 80aa42d6af..7740778adf 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapSecurityStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapSecurityStore.java @@ -29,12 +29,11 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.device.data.lwm2m.BootstrapConfiguration; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.transport.lwm2m.secure.TbLwM2MSecurityInfo; import org.thingsboard.server.transport.lwm2m.secure.LwM2mCredentialsSecurityInfoValidator; +import org.thingsboard.server.transport.lwm2m.secure.TbLwM2MSecurityInfo; import org.thingsboard.server.transport.lwm2m.server.LwM2mSessionMsgListener; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportContext; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper; -import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; import java.io.IOException; import java.security.GeneralSecurityException; @@ -46,6 +45,7 @@ import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.L import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_TELEMETRY; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.getBootstrapParametersFromThingsboard; +import static org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mTypeServer.BOOTSTRAP; @Slf4j @Service("LwM2MBootstrapSecurityStore") @@ -68,7 +68,7 @@ public class LwM2MBootstrapSecurityStore implements BootstrapSecurityStore { @Override public Iterator getAllByEndpoint(String endPoint) { - TbLwM2MSecurityInfo store = lwM2MCredentialsSecurityInfoValidator.getEndpointSecurityInfoByCredentialsId(endPoint, LwM2mTransportUtil.LwM2mTypeServer.BOOTSTRAP); + TbLwM2MSecurityInfo store = lwM2MCredentialsSecurityInfoValidator.getEndpointSecurityInfoByCredentialsId(endPoint, BOOTSTRAP); if (store.getBootstrapCredentialConfig() != null && store.getSecurityMode() != null) { /* add value to store from BootstrapJson */ this.setBootstrapConfigScurityInfo(store); @@ -92,7 +92,7 @@ public class LwM2MBootstrapSecurityStore implements BootstrapSecurityStore { @Override public SecurityInfo getByIdentity(String identity) { - TbLwM2MSecurityInfo store = lwM2MCredentialsSecurityInfoValidator.getEndpointSecurityInfoByCredentialsId(identity, LwM2mTransportUtil.LwM2mTypeServer.BOOTSTRAP); + TbLwM2MSecurityInfo store = lwM2MCredentialsSecurityInfoValidator.getEndpointSecurityInfoByCredentialsId(identity, BOOTSTRAP); if (store.getBootstrapCredentialConfig() != null && store.getSecurityMode() != null) { /* add value to store from BootstrapJson */ this.setBootstrapConfigScurityInfo(store); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java index 1ec48b31f5..8bd97a952e 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java @@ -33,7 +33,7 @@ import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportContext; -import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; +import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mTypeServer; import java.io.IOException; import java.security.GeneralSecurityException; @@ -45,6 +45,7 @@ import static org.eclipse.leshan.core.SecurityMode.NO_SEC; import static org.eclipse.leshan.core.SecurityMode.PSK; import static org.eclipse.leshan.core.SecurityMode.RPK; import static org.eclipse.leshan.core.SecurityMode.X509; +import static org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mTypeServer.BOOTSTRAP; @Slf4j @Component @@ -55,7 +56,7 @@ public class LwM2mCredentialsSecurityInfoValidator { private final LwM2mTransportContext context; private final LwM2MTransportServerConfig config; - public TbLwM2MSecurityInfo getEndpointSecurityInfoByCredentialsId(String credentialsId, LwM2mTransportUtil.LwM2mTypeServer keyValue) { + public TbLwM2MSecurityInfo getEndpointSecurityInfoByCredentialsId(String credentialsId, LwM2mTypeServer keyValue) { CountDownLatch latch = new CountDownLatch(1); final TbLwM2MSecurityInfo[] resultSecurityStore = new TbLwM2MSecurityInfo[1]; log.warn("001) [{}]", credentialsId); @@ -94,11 +95,11 @@ public class LwM2mCredentialsSecurityInfoValidator { * @param keyValue - * @return SecurityInfo */ - private TbLwM2MSecurityInfo createSecurityInfo(String endpoint, String jsonStr, LwM2mTransportUtil.LwM2mTypeServer keyValue) { + private TbLwM2MSecurityInfo createSecurityInfo(String endpoint, String jsonStr, LwM2mTypeServer keyValue) { TbLwM2MSecurityInfo result = new TbLwM2MSecurityInfo(); LwM2MCredentials credentials = JacksonUtil.fromString(jsonStr, LwM2MCredentials.class); if (credentials != null) { - if (keyValue.equals(LwM2mTransportUtil.LwM2mTypeServer.BOOTSTRAP)) { + if (keyValue.equals(BOOTSTRAP)) { result.setBootstrapCredentialConfig(credentials.getBootstrap()); if (LwM2MSecurityMode.PSK.equals(credentials.getClient().getSecurityConfigClientMode())) { PSKClientCredentials pskClientConfig = (PSKClientCredentials) credentials.getClient(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java index dcd7fa89bb..792ba131e8 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java @@ -42,7 +42,6 @@ import org.thingsboard.server.common.transport.util.SslUtil; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials; -import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; import org.thingsboard.server.transport.lwm2m.server.store.TbEditableSecurityStore; import org.thingsboard.server.transport.lwm2m.server.store.TbLwM2MDtlsSessionStore; @@ -57,6 +56,8 @@ import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.List; +import static org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mTypeServer.CLIENT; + @Slf4j @Component @TbLwM2mTransportComponent @@ -117,7 +118,7 @@ public class TbLwM2MDtlsCertificateVerifier implements NewAdvancedCertificateVer String strCert = SslUtil.getCertificateString(cert); String sha3Hash = EncryptionUtil.getSha3Hash(strCert); - TbLwM2MSecurityInfo securityInfo = securityInfoValidator.getEndpointSecurityInfoByCredentialsId(sha3Hash, LwM2mTransportUtil.LwM2mTypeServer.CLIENT); + TbLwM2MSecurityInfo securityInfo = securityInfoValidator.getEndpointSecurityInfoByCredentialsId(sha3Hash, CLIENT); ValidateDeviceCredentialsResponse msg = securityInfo != null ? securityInfo.getMsg() : null; if (msg != null && org.thingsboard.server.common.data.StringUtils.isNotEmpty(msg.getCredentials())) { LwM2MCredentials credentials = JacksonUtil.fromString(msg.getCredentials(), LwM2MCredentials.class); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java index 9425ef7891..c027267197 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/DefaultLwM2mTransportService.java @@ -65,7 +65,7 @@ import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_ECDHE import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_PSK_WITH_AES_128_CBC_SHA256; import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_PSK_WITH_AES_128_CCM_8; import static org.thingsboard.server.transport.lwm2m.server.LwM2mNetworkConfig.getCoapConfig; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FIRMWARE_UPDATE_COAP_RECOURSE; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FIRMWARE_UPDATE_COAP_RECOURSE; @Slf4j @Component diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java index f6456a394c..105943eab8 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportCoapResource.java @@ -31,8 +31,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FIRMWARE_UPDATE_COAP_RECOURSE; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.SOFTWARE_UPDATE_COAP_RECOURSE; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FIRMWARE_UPDATE_COAP_RECOURSE; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.SOFTWARE_UPDATE_COAP_RECOURSE; @Slf4j public class LwM2mTransportCoapResource extends AbstractLwM2mTransportResource { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java index d04e4993f1..64f80f7353 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java @@ -43,13 +43,13 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.device.data.lwm2m.BootstrapConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.ota.OtaPackageKey; -import org.thingsboard.server.common.data.ota.OtaPackageType; -import org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus; -import org.thingsboard.server.common.data.ota.OtaPackageUtil; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.ResourceValue; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateResultFw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateResultSw; +import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateStateSw; import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; import java.util.ArrayList; @@ -77,277 +77,20 @@ import static org.eclipse.leshan.core.model.ResourceModel.Type.STRING; import static org.eclipse.leshan.core.model.ResourceModel.Type.TIME; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_KEY; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.DOWNLOADED; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.DOWNLOADING; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.FAILED; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATED; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATING; -import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.VERIFIED; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_RESULT_ID; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_STATE_ID; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.SW_RESULT_ID; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.SW_UPDATE_STATE_ID; @Slf4j public class LwM2mTransportUtil { - public static final String EVENT_AWAKE = "AWAKE"; - public static final String RESPONSE_REQUEST_CHANNEL = "RESP_REQ"; - public static final String RESPONSE_CHANNEL = "RESP"; - public static final String OBSERVE_CHANNEL = "OBSERVE"; - - public static final String TRANSPORT_DEFAULT_LWM2M_VERSION = "1.0"; - public static final String CLIENT_LWM2M_SETTINGS = "clientLwM2mSettings"; - public static final String BOOTSTRAP = "bootstrap"; - public static final String SERVERS = "servers"; - public static final String LWM2M_SERVER = "lwm2mServer"; - public static final String BOOTSTRAP_SERVER = "bootstrapServer"; - public static final String OBSERVE_ATTRIBUTE_TELEMETRY = "observeAttr"; - public static final String ATTRIBUTE = "attribute"; - public static final String TELEMETRY = "telemetry"; - public static final String KEY_NAME = "keyName"; - public static final String OBSERVE_LWM2M = "observe"; - public static final String ATTRIBUTE_LWM2M = "attributeLwm2m"; - - private static final String REQUEST = "/request"; - private static final String ATTRIBUTES = "/" + ATTRIBUTE; - public static final String TELEMETRIES = "/" + TELEMETRY; - public static final String ATTRIBUTES_REQUEST = ATTRIBUTES + REQUEST; - public static final String DEVICE_ATTRIBUTES_REQUEST = ATTRIBUTES_REQUEST + "/"; - - public static final long DEFAULT_TIMEOUT = 2 * 60 * 1000L; // 2min in ms + public static final String LWM2M_VERSION_DEFAULT = "1.0"; public static final String LOG_LWM2M_TELEMETRY = "logLwm2m"; public static final String LOG_LWM2M_INFO = "info"; public static final String LOG_LWM2M_ERROR = "error"; public static final String LOG_LWM2M_WARN = "warn"; - public static final String LOG_LWM2M_VALUE = "value"; - - public static final String CLIENT_NOT_AUTHORIZED = "Client not authorized"; - public static final String LWM2M_VERSION_DEFAULT = "1.0"; - - // Firmware - public static final String FIRMWARE_UPDATE_COAP_RECOURSE = "tbfw"; - public static final String FW_UPDATE = "Firmware update"; - public static final Integer FW_5_ID = 5; - public static final Integer FW_19_ID = 19; - - // Package W - public static final String FW_PACKAGE_5_ID = "/5/0/0"; - public static final String FW_PACKAGE_19_ID = "/19/0/0"; - // Package URI - public static final String FW_PACKAGE_URI_ID = "/5/0/1"; - // State R - public static final String FW_STATE_ID = "/5/0/3"; - // Update Result R - public static final String FW_RESULT_ID = "/5/0/5"; - - public static final String FW_DELIVERY_METHOD = "/5/0/9"; - - // PkgName R - public static final String FW_NAME_ID = "/5/0/6"; - // PkgVersion R - public static final String FW_5_VER_ID = "/5/0/7"; - - /** - * Quectel@Hi15RM1-HLB_V1.0@BC68JAR01A10,V150R100C20B300SP7,V150R100C20B300SP7@8 - * BC68JAR01A10 - * # Request prodct type number - * ATI - * Quectel - * BC68 - * Revision:BC68JAR01A10 - */ - public static final String FW_3_VER_ID = "/3/0/3"; - // Update E - public static final String FW_UPDATE_ID = "/5/0/2"; - - // Software - public static final String SOFTWARE_UPDATE_COAP_RECOURSE = "softwareUpdateCoapRecourse"; - public static final String SW_UPDATE = "Software update"; - public static final Integer SW_ID = 9; - // Package W - public static final String SW_PACKAGE_ID = "/9/0/2"; - // Package URI - public static final String SW_PACKAGE_URI_ID = "/9/0/3"; - // Update State R - public static final String SW_UPDATE_STATE_ID = "/9/0/7"; - // Update Result R - public static final String SW_RESULT_ID = "/9/0/9"; - // PkgName R - public static final String SW_NAME_ID = "/9/0/0"; - // PkgVersion R - public static final String SW_VER_ID = "/9/0/1"; - // Install E - public static final String SW_INSTALL_ID = "/9/0/4"; - // Uninstall E - public static final String SW_UN_INSTALL_ID = "/9/0/6"; - - public enum LwM2mTypeServer { - BOOTSTRAP(0, "bootstrap"), - CLIENT(1, "client"); - - public int code; - public String type; - - LwM2mTypeServer(int code, String type) { - this.code = code; - this.type = type; - } - - public static LwM2mTypeServer fromLwM2mTypeServer(String type) { - for (LwM2mTypeServer sm : LwM2mTypeServer.values()) { - if (sm.type.equals(type)) { - return sm; - } - } - throw new IllegalArgumentException(String.format("Unsupported typeServer type : %d", type)); - } - } - - public static Optional toOtaPackageUpdateStatus(UpdateResultFw updateResultFw) { - switch (updateResultFw) { - case INITIAL: - return Optional.empty(); - case UPDATE_SUCCESSFULLY: - return Optional.of(UPDATED); - case NOT_ENOUGH: - case OUT_OFF_MEMORY: - case CONNECTION_LOST: - case INTEGRITY_CHECK_FAILURE: - case UNSUPPORTED_TYPE: - case INVALID_URI: - case UPDATE_FAILED: - case UNSUPPORTED_PROTOCOL: - return Optional.of(FAILED); - default: - throw new CodecException("Invalid value stateFw %s for FirmwareUpdateStatus.", updateResultFw.name()); - } - } - - public static Optional toOtaPackageUpdateStatus(UpdateStateFw updateStateFw) { - switch (updateStateFw) { - case IDLE: - return Optional.empty(); - case DOWNLOADING: - return Optional.of(DOWNLOADING); - case DOWNLOADED: - return Optional.of(DOWNLOADED); - case UPDATING: - return Optional.of(UPDATING); - default: - throw new CodecException("Invalid value stateFw %d for FirmwareUpdateStatus.", updateStateFw); - } - } - - /** - * SW Update State R - * 0: INITIAL Before downloading. (see 5.1.2.1) - * 1: DOWNLOAD STARTED The downloading process has started and is on-going. (see 5.1.2.2) - * 2: DOWNLOADED The package has been completely downloaded (see 5.1.2.3) - * 3: DELIVERED In that state, the package has been correctly downloaded and is ready to be installed. (see 5.1.2.4) - * If executing the Install Resource failed, the state remains at DELIVERED. - * If executing the Install Resource was successful, the state changes from DELIVERED to INSTALLED. - * After executing the UnInstall Resource, the state changes to INITIAL. - * 4: INSTALLED - */ - public enum UpdateStateSw { - INITIAL(0, "Initial"), - DOWNLOAD_STARTED(1, "DownloadStarted"), - DOWNLOADED(2, "Downloaded"), - DELIVERED(3, "Delivered"), - INSTALLED(4, "Installed"); - - public int code; - public String type; - - UpdateStateSw(int code, String type) { - this.code = code; - this.type = type; - } - - public static UpdateStateSw fromUpdateStateSwByType(String type) { - for (UpdateStateSw to : UpdateStateSw.values()) { - if (to.type.equals(type)) { - return to; - } - } - throw new IllegalArgumentException(String.format("Unsupported SW State type : %s", type)); - } - - public static UpdateStateSw fromUpdateStateSwByCode(int code) { - for (UpdateStateSw to : UpdateStateSw.values()) { - if (to.code == code) { - return to; - } - } - throw new IllegalArgumentException(String.format("Unsupported SW State type : %s", code)); - } - } - - /** - * SW Update Result - * Contains the result of downloading or installing/uninstalling the software - * 0: Initial value. - * - Prior to download any new package in the Device, Update Result MUST be reset to this initial value. - * - One side effect of executing the Uninstall resource is to reset Update Result to this initial value "0". - * 1: Downloading. - * - The package downloading process is on-going. - * 2: Software successfully installed. - * 3: Successfully Downloaded and package integrity verified - * (( 4-49, for expansion, of other scenarios)) - * ** Failed - * 50: Not enough storage for the new software package. - * 51: Out of memory during downloading process. - * 52: Connection lost during downloading process. - * 53: Package integrity check failure. - * 54: Unsupported package type. - * 56: Invalid URI - * 57: Device defined update error - * 58: Software installation failure - * 59: Uninstallation Failure during forUpdate(arg=0) - * 60-200 : (for expansion, selection to be in blocks depending on new introduction of features) - * This Resource MAY be reported by sending Observe operation. - */ - public enum UpdateResultSw { - INITIAL(0, "Initial value", false), - DOWNLOADING(1, "Downloading", false), - SUCCESSFULLY_INSTALLED(2, "Software successfully installed", false), - SUCCESSFULLY_DOWNLOADED_VERIFIED(3, "Successfully Downloaded and package integrity verified", false), - NOT_ENOUGH_STORAGE(50, "Not enough storage for the new software package", true), - OUT_OFF_MEMORY(51, "Out of memory during downloading process", true), - CONNECTION_LOST(52, "Connection lost during downloading process", false), - PACKAGE_CHECK_FAILURE(53, "Package integrity check failure.", false), - UNSUPPORTED_PACKAGE_TYPE(54, "Unsupported package type", false), - INVALID_URI(56, "Invalid URI", true), - UPDATE_ERROR(57, "Device defined update error", true), - INSTALL_FAILURE(58, "Software installation failure", true), - UN_INSTALL_FAILURE(59, "Uninstallation Failure during forUpdate(arg=0)", true); - - public int code; - public String type; - public boolean isAgain; - - UpdateResultSw(int code, String type, boolean isAgain) { - this.code = code; - this.type = type; - this.isAgain = isAgain; - } - - public static UpdateResultSw fromUpdateResultSwByType(String type) { - for (UpdateResultSw to : UpdateResultSw.values()) { - if (to.type.equals(type)) { - return to; - } - } - throw new IllegalArgumentException(String.format("Unsupported SW Update Result type : %s", type)); - } - - public static UpdateResultSw fromUpdateResultSwByCode(int code) { - for (UpdateResultSw to : UpdateResultSw.values()) { - if (to.code == code) { - return to; - } - } - throw new IllegalArgumentException(String.format("Unsupported SW Update Result code : %s", code)); - } - } public enum LwM2MClientStrategy { CLIENT_STRATEGY_1(1, "Read only resources marked as observation"), @@ -380,45 +123,7 @@ public class LwM2mTransportUtil { } } - /** - * FirmwareUpdateStatus { - * DOWNLOADING, DOWNLOADED, VERIFIED, UPDATING, UPDATED, FAILED - */ - public static OtaPackageUpdateStatus EqualsSwSateToFirmwareUpdateStatus(UpdateStateSw updateStateSw, UpdateResultSw updateResultSw) { - switch (updateResultSw) { - case INITIAL: - switch (updateStateSw) { - case INITIAL: - case DOWNLOAD_STARTED: - return DOWNLOADING; - case DOWNLOADED: - return DOWNLOADED; - case DELIVERED: - return VERIFIED; - } - case DOWNLOADING: - return DOWNLOADING; - case SUCCESSFULLY_INSTALLED: - return UPDATED; - case SUCCESSFULLY_DOWNLOADED_VERIFIED: - return VERIFIED; - case NOT_ENOUGH_STORAGE: - case OUT_OFF_MEMORY: - case CONNECTION_LOST: - case PACKAGE_CHECK_FAILURE: - case UNSUPPORTED_PACKAGE_TYPE: - case INVALID_URI: - case UPDATE_ERROR: - case INSTALL_FAILURE: - case UN_INSTALL_FAILURE: - return FAILED; - default: - throw new CodecException("Invalid value stateFw %s %s for FirmwareUpdateStatus.", updateStateSw.name(), updateResultSw.name()); - } - } - - - public static boolean equalsResourceValue(Object valueOld, Object valueNew, ResourceModel.Type type, LwM2mPath + public static boolean equalsResourceValue(Object valueOld, Object valueNew, ResourceModel.Type type, LwM2mPath resourcePath) throws CodecException { switch (type) { case BOOLEAN: @@ -477,18 +182,6 @@ public class LwM2mTransportUtil { return null; } -// public static LwM2mClientProfile getNewProfileParameters(JsonObject profilesConfigData, TenantId tenantId) { -// LwM2mClientProfile lwM2MClientProfile = new LwM2mClientProfile(); -// lwM2MClientProfile.setTenantId(tenantId); -// lwM2MClientProfile.setPostClientLwM2mSettings(profilesConfigData.get(CLIENT_LWM2M_SETTINGS).getAsJsonObject()); -// lwM2MClientProfile.setPostKeyNameProfile(profilesConfigData.get(OBSERVE_ATTRIBUTE_TELEMETRY).getAsJsonObject().get(KEY_NAME).getAsJsonObject()); -// lwM2MClientProfile.setPostAttributeProfile(profilesConfigData.get(OBSERVE_ATTRIBUTE_TELEMETRY).getAsJsonObject().get(ATTRIBUTE).getAsJsonArray()); -// lwM2MClientProfile.setPostTelemetryProfile(profilesConfigData.get(OBSERVE_ATTRIBUTE_TELEMETRY).getAsJsonObject().get(TELEMETRY).getAsJsonArray()); -// lwM2MClientProfile.setPostObserveProfile(profilesConfigData.get(OBSERVE_ATTRIBUTE_TELEMETRY).getAsJsonObject().get(OBSERVE_LWM2M).getAsJsonArray()); -// lwM2MClientProfile.setPostAttributeLwm2mProfile(profilesConfigData.get(OBSERVE_ATTRIBUTE_TELEMETRY).getAsJsonObject().get(ATTRIBUTE_LWM2M).getAsJsonObject()); -// return lwM2MClientProfile; -// } - public static Lwm2mDeviceProfileTransportConfiguration toLwM2MClientProfile(DeviceProfile deviceProfile) { DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); if (transportConfiguration.getType().equals(DeviceTransportType.LWM2M)) { @@ -603,7 +296,6 @@ public class LwM2mTransportUtil { if (keyArray.length > 1 && keyArray[1].split(LWM2M_SEPARATOR_KEY).length == 2) { return pathIdVer; } else { - LwM2mPath pathObjId = new LwM2mPath(pathIdVer); return convertObjectIdToVersionedId(pathIdVer, registration); } } @@ -743,18 +435,18 @@ public class LwM2mTransportUtil { } } - public static boolean isFwSwWords(String pathName) { - return OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.TITLE).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.CHECKSUM).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.CHECKSUM_ALGORITHM).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.SIZE).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.VERSION).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.TITLE).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.CHECKSUM).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.CHECKSUM_ALGORITHM).equals(pathName) - || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.SIZE).equals(pathName); - } +// public static boolean isFwSwWords(String pathName) { +// return OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.TITLE).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.CHECKSUM).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.CHECKSUM_ALGORITHM).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.SIZE).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.VERSION).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.TITLE).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.CHECKSUM).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.CHECKSUM_ALGORITHM).equals(pathName) +// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.SIZE).equals(pathName); +// } /** * @param lwM2MClient - diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java index 0e71a1e865..dbcb82a167 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java @@ -33,12 +33,10 @@ import org.eclipse.leshan.server.security.SecurityInfo; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.transport.lwm2m.server.LwM2mQueuedRequest; -import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import java.util.Collection; import java.util.Map; @@ -53,7 +51,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.TRANSPORT_DEFAULT_LWM2M_VERSION; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LWM2M_VERSION_DEFAULT; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.equalsResourceTypeGetSimpleName; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.fromVersionedIdToObjectId; @@ -307,7 +305,7 @@ public class LwM2mClient implements Cloneable { LwM2mPath pathIds = new LwM2mPath(fromVersionedIdToObjectId(path)); String verSupportedObject = registration.getSupportedObject().get(pathIds.getObjectId()); String verRez = getVerFromPathIdVerOrId(path); - return verRez == null ? TRANSPORT_DEFAULT_LWM2M_VERSION.equals(verSupportedObject) : verRez.equals(verSupportedObject); + return verRez == null ? LWM2M_VERSION_DEFAULT.equals(verSupportedObject) : verRez.equals(verSupportedObject); } /** diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java index 4126541176..8f0a9da33f 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.transport.lwm2m.server.ota; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.eclipse.leshan.core.node.codec.CodecException; import org.eclipse.leshan.core.request.ContentFormat; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -32,12 +33,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; -import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.LwM2MSoftwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportServerHelper; -import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; -import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; -import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; import org.thingsboard.server.transport.lwm2m.server.attributes.LwM2MAttributesService; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; @@ -48,6 +44,13 @@ import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MExecuteRequ import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplaceRequest; import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.LwM2MFirmwareUpdateStrategy; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateDeliveryFw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateResultFw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.ota.software.LwM2MSoftwareUpdateStrategy; +import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateResultSw; +import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateStateSw; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import javax.annotation.PostConstruct; @@ -60,8 +63,13 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import static org.thingsboard.server.common.data.ota.OtaPackageKey.STATE; +import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.DOWNLOADED; +import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.DOWNLOADING; +import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.FAILED; +import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATED; +import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATING; +import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.VERIFIED; import static org.thingsboard.server.common.data.ota.OtaPackageUtil.getAttributeKey; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FIRMWARE_UPDATE_COAP_RECOURSE; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_TELEMETRY; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; @@ -78,13 +86,30 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl public static final String SOFTWARE_TITLE = getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.TITLE); public static final String SOFTWARE_URL = getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.URL); + public static final String FIRMWARE_UPDATE_COAP_RECOURSE = "tbfw"; + public static final String SOFTWARE_UPDATE_COAP_RECOURSE = "tbsw"; private static final String FW_PACKAGE_5_ID = "/5/0/0"; private static final String FW_URL_ID = "/5/0/1"; private static final String FW_EXECUTE_ID = "/5/0/2"; - private static final String FW_NAME_ID = "/5/0/6"; - private static final String FW_VER_ID = "/5/0/7"; + public static final String FW_STATE_ID = "/5/0/3"; + public static final String FW_RESULT_ID = "/5/0/5"; + public static final String FW_NAME_ID = "/5/0/6"; + public static final String FW_5_VER_ID = "/5/0/7"; + /** + * Quectel@Hi15RM1-HLB_V1.0@BC68JAR01A10,V150R100C20B300SP7,V150R100C20B300SP7@8 + * Revision:BC68JAR01A10 + */ + public static final String FW_3_VER_ID = "/3/0/3"; + public static final String FW_DELIVERY_METHOD = "/5/0/9"; + private static final String SW_NAME_ID = "/9/0/0"; private static final String SW_VER_ID = "/9/0/1"; + public static final String SW_PACKAGE_ID = "/9/0/2"; + public static final String SW_PACKAGE_URI_ID = "/9/0/3"; + public static final String SW_INSTALL_ID = "/9/0/4"; + public static final String SW_UPDATE_STATE_ID = "/9/0/7"; + public static final String SW_RESULT_ID = "/9/0/9"; + public static final String SW_UN_INSTALL_ID = "/9/0/6"; private final Map fwStates = new ConcurrentHashMap<>(); private final Map swStates = new ConcurrentHashMap<>(); @@ -216,7 +241,7 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl executeFwUpdate(client); } fwInfo.setUpdateState(state); - Optional status = LwM2mTransportUtil.toOtaPackageUpdateStatus(state); + Optional status = this.toOtaPackageUpdateStatus(state); status.ifPresent(otaStatus -> sendStateUpdateToTelemetry(client, fwInfo, otaStatus, "Firmware Update State: " + state.name())); } @@ -226,7 +251,7 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl log.debug("[{}] Current fw result: {}", client.getEndpoint(), code); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); UpdateResultFw result = UpdateResultFw.fromUpdateResultFwByCode(code.intValue()); - Optional status = LwM2mTransportUtil.toOtaPackageUpdateStatus(result); + Optional status = this.toOtaPackageUpdateStatus(result); status.ifPresent(otaStatus -> sendStateUpdateToTelemetry(client, fwInfo, otaStatus, "Firmware Update Result: " + result.name())); if (result.isAgain() && fwInfo.getRetryAttempts() <= 2) { @@ -299,10 +324,10 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus())) { UUID otaPackageId = new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB()); LwM2MFirmwareUpdateStrategy strategy; - if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == 2) { + if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == UpdateDeliveryFw.BOTH.code) { strategy = fwInfo.getFwStrategy(); } else { - strategy = fwInfo.getDeliveryMethod() == 0 ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; + strategy = fwInfo.getDeliveryMethod() == UpdateDeliveryFw.PULL.code ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; } switch (strategy) { case OBJ_5_BINARY: @@ -380,4 +405,78 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl helper.sendParametersOnThingsboardTelemetry(result, client.getSession()); } + private static Optional toOtaPackageUpdateStatus(UpdateResultFw updateResultFw) { + switch (updateResultFw) { + case INITIAL: + return Optional.empty(); + case UPDATE_SUCCESSFULLY: + return Optional.of(UPDATED); + case NOT_ENOUGH: + case OUT_OFF_MEMORY: + case CONNECTION_LOST: + case INTEGRITY_CHECK_FAILURE: + case UNSUPPORTED_TYPE: + case INVALID_URI: + case UPDATE_FAILED: + case UNSUPPORTED_PROTOCOL: + return Optional.of(FAILED); + default: + throw new CodecException("Invalid value stateFw %s for FirmwareUpdateStatus.", updateResultFw.name()); + } + } + + private static Optional toOtaPackageUpdateStatus(UpdateStateFw updateStateFw) { + switch (updateStateFw) { + case IDLE: + return Optional.empty(); + case DOWNLOADING: + return Optional.of(DOWNLOADING); + case DOWNLOADED: + return Optional.of(DOWNLOADED); + case UPDATING: + return Optional.of(UPDATING); + default: + throw new CodecException("Invalid value stateFw %d for FirmwareUpdateStatus.", updateStateFw); + } + } + + /** + * FirmwareUpdateStatus { + * DOWNLOADING, DOWNLOADED, VERIFIED, UPDATING, UPDATED, FAILED + */ + public static Optional toSwSateResultUpdateStatus(UpdateStateSw updateStateSw, UpdateResultSw updateResultSw) { + switch (updateResultSw) { + case INITIAL: + switch (updateStateSw) { + case INITIAL: + case DOWNLOAD_STARTED: + return Optional.of(DOWNLOADING); + case DOWNLOADED: + return Optional.of(DOWNLOADED); + case DELIVERED: + return Optional.of(VERIFIED); + } + case DOWNLOADING: + return Optional.of(DOWNLOADING); + case SUCCESSFULLY_INSTALLED: + return Optional.of(UPDATED); + case SUCCESSFULLY_DOWNLOADED_VERIFIED: + return Optional.of(VERIFIED); + case NOT_ENOUGH_STORAGE: + case OUT_OFF_MEMORY: + case CONNECTION_LOST: + case PACKAGE_CHECK_FAILURE: + case UNSUPPORTED_PACKAGE_TYPE: + case INVALID_URI: + case UPDATE_ERROR: + case INSTALL_FAILURE: + case UN_INSTALL_FAILURE: + return Optional.of(FAILED); + default: + throw new CodecException("Invalid value stateFw %s %s for FirmwareUpdateStatus.", updateStateSw.name(), updateResultSw.name()); + } + } + + + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java index 74bf8d1fac..98e2055fdc 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java @@ -18,10 +18,10 @@ package org.thingsboard.server.transport.lwm2m.server.ota; import lombok.Data; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageType; -import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.LwM2MSoftwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.UpdateResultFw; -import org.thingsboard.server.transport.lwm2m.server.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.LwM2MFirmwareUpdateStrategy; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateResultFw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.ota.software.LwM2MSoftwareUpdateStrategy; import java.util.Optional; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MFirmwareUpdateStrategy.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/LwM2MFirmwareUpdateStrategy.java similarity index 96% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MFirmwareUpdateStrategy.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/LwM2MFirmwareUpdateStrategy.java index f111152b76..d8ce5b1fbb 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MFirmwareUpdateStrategy.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/LwM2MFirmwareUpdateStrategy.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.lwm2m.server; +package org.thingsboard.server.transport.lwm2m.server.ota.firmware; public enum LwM2MFirmwareUpdateStrategy { OBJ_5_BINARY(1, "ObjectId 5, Binary"), diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateDeliveryFw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateDeliveryFw.java new file mode 100644 index 0000000000..dcb93a5283 --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateDeliveryFw.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.ota.firmware; + +/** + * /** Delivery Method R + * 0: Pull only + * 1: Push only + * 2: Both: + * - In this case the LwM2M Server MAY choose the preferred mechanism for conveying the firmware image to the LwM2M Client. + */ +public enum UpdateDeliveryFw { + PULL(0, "Pull only"), + PUSH(1, "Push only"), + BOTH(2, "Push or Push"); + + public int code; + public String type; + + UpdateDeliveryFw(int code, String type) { + this.code = code; + this.type = type; + } + + public static UpdateDeliveryFw fromStateFwByType(String type) { + for (UpdateDeliveryFw to : UpdateDeliveryFw.values()) { + if (to.type.equals(type)) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported FW delivery type : %s", type)); + } + + public static UpdateDeliveryFw fromStateFwByCode(int code) { + for (UpdateDeliveryFw to : UpdateDeliveryFw.values()) { + if (to.code == code) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported FW delivery code : %s", code)); + } +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateResultFw.java similarity index 97% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateResultFw.java index 79361aeb1d..5a71f41bc5 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateResultFw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateResultFw.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.lwm2m.server; +package org.thingsboard.server.transport.lwm2m.server.ota.firmware; import lombok.Getter; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateStateFw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateStateFw.java similarity index 96% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateStateFw.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateStateFw.java index b47a96e9bd..447749fd50 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/UpdateStateFw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateStateFw.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.lwm2m.server; +package org.thingsboard.server.transport.lwm2m.server.ota.firmware; /** * /** State R diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MSoftwareUpdateStrategy.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/LwM2MSoftwareUpdateStrategy.java similarity index 95% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MSoftwareUpdateStrategy.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/LwM2MSoftwareUpdateStrategy.java index 0df3f5ca2d..5fbdc25a2f 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2MSoftwareUpdateStrategy.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/LwM2MSoftwareUpdateStrategy.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.lwm2m.server; +package org.thingsboard.server.transport.lwm2m.server.ota.software; public enum LwM2MSoftwareUpdateStrategy { BINARY(1, "ObjectId 9, Binary"), diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateResultSw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateResultSw.java new file mode 100644 index 0000000000..22c92205d0 --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateResultSw.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.ota.software; + +/** + * SW Update Result + * Contains the result of downloading or installing/uninstalling the software + * 0: Initial value. + * - Prior to download any new package in the Device, Update Result MUST be reset to this initial value. + * - One side effect of executing the Uninstall resource is to reset Update Result to this initial value "0". + * 1: Downloading. + * - The package downloading process is on-going. + * 2: Software successfully installed. + * 3: Successfully Downloaded and package integrity verified + * (( 4-49, for expansion, of other scenarios)) + * ** Failed + * 50: Not enough storage for the new software package. + * 51: Out of memory during downloading process. + * 52: Connection lost during downloading process. + * 53: Package integrity check failure. + * 54: Unsupported package type. + * 56: Invalid URI + * 57: Device defined update error + * 58: Software installation failure + * 59: Uninstallation Failure during forUpdate(arg=0) + * 60-200 : (for expansion, selection to be in blocks depending on new introduction of features) + * This Resource MAY be reported by sending Observe operation. + */ +public enum UpdateResultSw { + INITIAL(0, "Initial value", false), + DOWNLOADING(1, "Downloading", false), + SUCCESSFULLY_INSTALLED(2, "Software successfully installed", false), + SUCCESSFULLY_DOWNLOADED_VERIFIED(3, "Successfully Downloaded and package integrity verified", false), + NOT_ENOUGH_STORAGE(50, "Not enough storage for the new software package", true), + OUT_OFF_MEMORY(51, "Out of memory during downloading process", true), + CONNECTION_LOST(52, "Connection lost during downloading process", false), + PACKAGE_CHECK_FAILURE(53, "Package integrity check failure.", false), + UNSUPPORTED_PACKAGE_TYPE(54, "Unsupported package type", false), + INVALID_URI(56, "Invalid URI", true), + UPDATE_ERROR(57, "Device defined update error", true), + INSTALL_FAILURE(58, "Software installation failure", true), + UN_INSTALL_FAILURE(59, "Uninstallation Failure during forUpdate(arg=0)", true); + + public int code; + public String type; + public boolean isAgain; + + UpdateResultSw(int code, String type, boolean isAgain) { + this.code = code; + this.type = type; + this.isAgain = isAgain; + } + + public static UpdateResultSw fromUpdateResultSwByType(String type) { + for (UpdateResultSw to : UpdateResultSw.values()) { + if (to.type.equals(type)) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported SW Update Result type : %s", type)); + } + + public static UpdateResultSw fromUpdateResultSwByCode(int code) { + for (UpdateResultSw to : UpdateResultSw.values()) { + if (to.code == code) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported SW Update Result code : %s", code)); + } +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateStateSw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateStateSw.java new file mode 100644 index 0000000000..a5f772a11d --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateStateSw.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.ota.software; + +/** + * SW Update State R + * 0: INITIAL Before downloading. (see 5.1.2.1) + * 1: DOWNLOAD STARTED The downloading process has started and is on-going. (see 5.1.2.2) + * 2: DOWNLOADED The package has been completely downloaded (see 5.1.2.3) + * 3: DELIVERED In that state, the package has been correctly downloaded and is ready to be installed. (see 5.1.2.4) + * If executing the Install Resource failed, the state remains at DELIVERED. + * If executing the Install Resource was successful, the state changes from DELIVERED to INSTALLED. + * After executing the UnInstall Resource, the state changes to INITIAL. + * 4: INSTALLED + */ +public enum UpdateStateSw { + INITIAL(0, "Initial"), + DOWNLOAD_STARTED(1, "DownloadStarted"), + DOWNLOADED(2, "Downloaded"), + DELIVERED(3, "Delivered"), + INSTALLED(4, "Installed"); + + public int code; + public String type; + + UpdateStateSw(int code, String type) { + this.code = code; + this.type = type; + } + + public static UpdateStateSw fromUpdateStateSwByType(String type) { + for (UpdateStateSw to : UpdateStateSw.values()) { + if (to.type.equals(type)) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported SW State type : %s", type)); + } + + public static UpdateStateSw fromUpdateStateSwByCode(int code) { + for (UpdateStateSw to : UpdateStateSw.values()) { + if (to.code == code) { + return to; + } + } + throw new IllegalArgumentException(String.format("Unsupported SW State type : %s", code)); + } +} + diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java index 293e68ec39..d47be49978 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mSecurityStore.java @@ -16,20 +16,13 @@ package org.thingsboard.server.transport.lwm2m.server.store; import lombok.extern.slf4j.Slf4j; -import org.eclipse.leshan.server.security.EditableSecurityStore; import org.eclipse.leshan.server.security.NonUniqueSecurityInfoException; import org.eclipse.leshan.server.security.SecurityInfo; -import org.eclipse.leshan.server.security.SecurityStore; -import org.eclipse.leshan.server.security.SecurityStoreListener; import org.jetbrains.annotations.Nullable; -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.secure.LwM2mCredentialsSecurityInfoValidator; import org.thingsboard.server.transport.lwm2m.secure.TbLwM2MSecurityInfo; -import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil; -import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientContext; -import java.util.Collection; +import static org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mTypeServer.CLIENT; @Slf4j public class TbLwM2mSecurityStore implements TbEditableSecurityStore { @@ -67,7 +60,7 @@ public class TbLwM2mSecurityStore implements TbEditableSecurityStore { @Nullable public SecurityInfo fetchAndPutSecurityInfo(String credentialsId) { - TbLwM2MSecurityInfo securityInfo = validator.getEndpointSecurityInfoByCredentialsId(credentialsId, LwM2mTransportUtil.LwM2mTypeServer.CLIENT); + TbLwM2MSecurityInfo securityInfo = validator.getEndpointSecurityInfoByCredentialsId(credentialsId, CLIENT); try { if (securityInfo != null) { securityStore.put(securityInfo); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index cf158aa14e..3307db2fca 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -52,8 +52,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; -import org.thingsboard.server.transport.lwm2m.server.LwM2MFirmwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.LwM2MSoftwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.LwM2mOtaConvert; import org.thingsboard.server.transport.lwm2m.server.LwM2mQueuedRequest; import org.thingsboard.server.transport.lwm2m.server.LwM2mSessionMsgListener; @@ -83,6 +81,8 @@ import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteAttrib import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteAttributesRequest; import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.ota.LwM2MOtaUpdateService; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.LwM2MFirmwareUpdateStrategy; +import org.thingsboard.server.transport.lwm2m.server.ota.software.LwM2MSoftwareUpdateStrategy; import org.thingsboard.server.transport.lwm2m.server.rpc.LwM2MRpcRequestHandler; import org.thingsboard.server.transport.lwm2m.server.store.TbLwM2MDtlsSessionStore; import org.thingsboard.server.transport.lwm2m.utils.LwM2mValueConverterImpl; @@ -105,18 +105,18 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_3_VER_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_5_VER_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_DELIVERY_METHOD; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_NAME_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_RESULT_ID; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.FW_STATE_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_ERROR; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_INFO; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.LOG_LWM2M_WARN; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertObjectIdToVersionedId; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.convertOtaUpdateValueToString; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportUtil.fromVersionedIdToObjectId; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_3_VER_ID; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_5_VER_ID; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_DELIVERY_METHOD; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_NAME_ID; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_RESULT_ID; +import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_STATE_ID; @Slf4j diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mTypeServer.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mTypeServer.java new file mode 100644 index 0000000000..0ecd59f41c --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/LwM2mTypeServer.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.uplink; + +public enum LwM2mTypeServer { + BOOTSTRAP(0, "bootstrap"), + CLIENT(1, "client"); + + public int code; + public String type; + + LwM2mTypeServer(int code, String type) { + this.code = code; + this.type = type; + } + + public static LwM2mTypeServer fromLwM2mTypeServer(String type) { + for (LwM2mTypeServer sm : LwM2mTypeServer.values()) { + if (sm.type.equals(type)) { + return sm; + } + } + throw new IllegalArgumentException(String.format("Unsupported typeServer type : %d", type)); + } +} + From 1f9b4c093adc294d780030caa3993589b37e4edb Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 22 Jun 2021 14:27:56 +0300 Subject: [PATCH 100/101] Update getOAuth2Clients Rest client method --- .../java/org/thingsboard/rest/client/RestClient.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 6fbf28f24e..2d0914c7d4 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -104,6 +104,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; import org.thingsboard.server.common.data.oauth2.OAuth2Info; +import org.thingsboard.server.common.data.oauth2.PlatformType; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -1772,7 +1773,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { }).getBody(); } - public List getOAuth2Clients(String pkgName) { + public List getOAuth2Clients(String pkgName, PlatformType platformType) { Map params = new HashMap<>(); StringBuilder urlBuilder = new StringBuilder(baseURL); urlBuilder.append("/api/noauth/oauth2Clients"); @@ -1780,6 +1781,15 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { urlBuilder.append("?pkgName={pkgName}"); params.put("pkgName", pkgName); } + if (platformType != null) { + if (pkgName != null) { + urlBuilder.append("&"); + } else { + urlBuilder.append("?"); + } + urlBuilder.append("platform={platform}"); + params.put("platform", platformType.name()); + } return restTemplate.exchange( urlBuilder.toString(), HttpMethod.POST, From e83064ece73ec11726c131e56c47949f6d70e7e2 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 22 Jun 2021 15:53:06 +0300 Subject: [PATCH 101/101] Refactoring of the LwM2MTransport --- application/src/test/resources/logback.xml | 2 +- .../coap/OtaPackageTransportResource.java | 42 +------------ .../config/LwM2MTransportServerConfig.java | 2 +- ...LwM2mCredentialsSecurityInfoValidator.java | 5 +- .../lwm2m/server/LwM2mTransportUtil.java | 29 +++------ .../ota/DefaultLwM2MOtaUpdateService.java | 59 +++++++++---------- .../lwm2m/server/ota/LwM2MClientOtaInfo.java | 10 ++-- .../server/ota/LwM2MOtaUpdateService.java | 5 +- ...eryFw.java => FirmwareDeliveryMethod.java} | 19 ++---- ...esultFw.java => FirmwareUpdateResult.java} | 12 ++-- ...eStateFw.java => FirmwareUpdateState.java} | 12 ++-- ...esultSw.java => SoftwareUpdateResult.java} | 12 ++-- ...eStateSw.java => SoftwareUpdateState.java} | 12 ++-- .../uplink/DefaultLwM2MUplinkMsgHandler.java | 44 +++++--------- 14 files changed, 98 insertions(+), 167 deletions(-) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/{UpdateDeliveryFw.java => FirmwareDeliveryMethod.java} (70%) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/{UpdateResultFw.java => FirmwareUpdateResult.java} (86%) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/{UpdateStateFw.java => FirmwareUpdateState.java} (80%) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/{UpdateResultSw.java => SoftwareUpdateResult.java} (89%) rename common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/{UpdateStateSw.java => SoftwareUpdateState.java} (84%) diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 69e5d98193..d3301bf660 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -14,7 +14,7 @@ - + diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java index 224dd61e2a..2aadea26b2 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java @@ -42,14 +42,12 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { private static final int ACCESS_TOKEN_POSITION = 2; private final OtaPackageType otaPackageType; - private final ExecutorService sendOtaDataOutUriLarge; public OtaPackageTransportResource(CoapTransportContext ctx, OtaPackageType otaPackageType) { super(ctx, otaPackageType.getKeyPrefix()); this.otaPackageType = otaPackageType; this.setObservable(true); - this.sendOtaDataOutUriLarge = ThingsBoardExecutors.newWorkStealingPool(10, "LwM2M sendOtaDataOutUriLarge"); } @Override @@ -140,46 +138,12 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { Response response = new Response(CoAP.ResponseCode.CONTENT); if (data != null && data.length > 0) { response.setPayload(data); -// response.getOptions().setAccept(MediaTypeRegistry.APPLICATION_OCTET_STREAM); if (exchange.getRequestOptions().getBlock2() != null) { int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); boolean lastFlag = data.length <= chunkSize; - this.sendOtaDataOutUriLarge.submit(() -> { - response.getOptions().setBlock2(chunkSize, lastFlag, 0); - }); } - exchange.respond(response); - } - } - - public class CoapResourceObserver implements ResourceObserver { - @Override - public void changedName(String old) { - - } - - @Override - public void changedPath(String old) { - - } - - @Override - public void addedChild(Resource child) { - - } - - @Override - public void removedChild(Resource child) { - - } - - @Override - public void addedObserveRelation(ObserveRelation relation) { - - } - - @Override - public void removedObserveRelation(ObserveRelation relation) { - + response.getOptions().setBlock2(chunkSize, lastFlag, 0); + } + transportContext.getExecutor().submit(() -> exchange.respond(response)); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java index a2ff361712..730effaf3c 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java @@ -142,7 +142,7 @@ public class LwM2MTransportServerConfig implements LwM2MSecureServerConfig { URI uri = null; try { uri = Resources.getResource(keyStorePathFile).toURI(); - log.error("URI: {}", uri); + log.info("URI: {}", uri); File keyStoreFile = new File(uri); InputStream inKeyStore = new FileInputStream(keyStoreFile); keyStoreValue = KeyStore.getInstance(keyStoreType); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java index 8bd97a952e..d11f39c7f2 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/LwM2mCredentialsSecurityInfoValidator.java @@ -59,12 +59,12 @@ public class LwM2mCredentialsSecurityInfoValidator { public TbLwM2MSecurityInfo getEndpointSecurityInfoByCredentialsId(String credentialsId, LwM2mTypeServer keyValue) { CountDownLatch latch = new CountDownLatch(1); final TbLwM2MSecurityInfo[] resultSecurityStore = new TbLwM2MSecurityInfo[1]; - log.warn("001) [{}]", credentialsId); + log.trace("Validating credentials [{}]", credentialsId); context.getTransportService().process(ValidateDeviceLwM2MCredentialsRequestMsg.newBuilder().setCredentialsId(credentialsId).build(), new TransportServiceCallback<>() { @Override public void onSuccess(ValidateDeviceCredentialsResponse msg) { - log.warn("002) [{}] [{}]", credentialsId, msg); + log.trace("Validated credentials: [{}] [{}]", credentialsId, msg); String credentialsBody = msg.getCredentials(); resultSecurityStore[0] = createSecurityInfo(credentialsId, credentialsBody, keyValue); resultSecurityStore[0].setMsg(msg); @@ -74,7 +74,6 @@ public class LwM2mCredentialsSecurityInfoValidator { @Override public void onError(Throwable e) { - log.warn("003) [{}] [{}] Failed to process credentials ", credentialsId, e); log.trace("[{}] [{}] Failed to process credentials ", credentialsId, e); resultSecurityStore[0] = createSecurityInfo(credentialsId, null, null); latch.countDown(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java index 64f80f7353..bbab28ded5 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportUtil.java @@ -46,10 +46,10 @@ import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTrans import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.ResourceValue; -import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateResultFw; -import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateStateFw; -import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateResultSw; -import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateStateSw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.FirmwareUpdateResult; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.FirmwareUpdateState; +import org.thingsboard.server.transport.lwm2m.server.ota.software.SoftwareUpdateResult; +import org.thingsboard.server.transport.lwm2m.server.ota.software.SoftwareUpdateState; import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2MUplinkMsgHandler; import java.util.ArrayList; @@ -148,19 +148,19 @@ public class LwM2mTransportUtil { if (path != null) { if (FW_STATE_ID.equals(path)) { lwM2mOtaConvert.setCurrentType(STRING); - lwM2mOtaConvert.setValue(UpdateStateFw.fromStateFwByCode(((Long) value).intValue()).type); + lwM2mOtaConvert.setValue(FirmwareUpdateState.fromStateFwByCode(((Long) value).intValue()).type); return lwM2mOtaConvert; } else if (FW_RESULT_ID.equals(path)) { lwM2mOtaConvert.setCurrentType(STRING); - lwM2mOtaConvert.setValue(UpdateResultFw.fromUpdateResultFwByCode(((Long) value).intValue()).getType()); + lwM2mOtaConvert.setValue(FirmwareUpdateResult.fromUpdateResultFwByCode(((Long) value).intValue()).getType()); return lwM2mOtaConvert; } else if (SW_UPDATE_STATE_ID.equals(path)) { lwM2mOtaConvert.setCurrentType(STRING); - lwM2mOtaConvert.setValue(UpdateStateSw.fromUpdateStateSwByCode(((Long) value).intValue()).type); + lwM2mOtaConvert.setValue(SoftwareUpdateState.fromUpdateStateSwByCode(((Long) value).intValue()).type); return lwM2mOtaConvert; } else if (SW_RESULT_ID.equals(path)) { lwM2mOtaConvert.setCurrentType(STRING); - lwM2mOtaConvert.setValue(UpdateResultSw.fromUpdateResultSwByCode(((Long) value).intValue()).type); + lwM2mOtaConvert.setValue(SoftwareUpdateResult.fromUpdateResultSwByCode(((Long) value).intValue()).type); return lwM2mOtaConvert; } } @@ -435,19 +435,6 @@ public class LwM2mTransportUtil { } } -// public static boolean isFwSwWords(String pathName) { -// return OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.VERSION).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.TITLE).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.CHECKSUM).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.CHECKSUM_ALGORITHM).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.FIRMWARE, OtaPackageKey.SIZE).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.VERSION).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.TITLE).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.CHECKSUM).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.CHECKSUM_ALGORITHM).equals(pathName) -// || OtaPackageUtil.getAttributeKey(OtaPackageType.SOFTWARE, OtaPackageKey.SIZE).equals(pathName); -// } - /** * @param lwM2MClient - * @param path - diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java index 8f0a9da33f..d3f1e8eb9d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java @@ -25,6 +25,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.device.data.lwm2m.OtherConfiguration; import org.thingsboard.server.common.data.ota.OtaPackageKey; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus; @@ -45,12 +46,12 @@ import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteReplac import org.thingsboard.server.transport.lwm2m.server.downlink.TbLwM2MWriteResponseCallback; import org.thingsboard.server.transport.lwm2m.server.log.LwM2MTelemetryLogService; import org.thingsboard.server.transport.lwm2m.server.ota.firmware.LwM2MFirmwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateDeliveryFw; -import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateResultFw; -import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.FirmwareDeliveryMethod; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.FirmwareUpdateResult; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.FirmwareUpdateState; import org.thingsboard.server.transport.lwm2m.server.ota.software.LwM2MSoftwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateResultSw; -import org.thingsboard.server.transport.lwm2m.server.ota.software.UpdateStateSw; +import org.thingsboard.server.transport.lwm2m.server.ota.software.SoftwareUpdateResult; +import org.thingsboard.server.transport.lwm2m.server.ota.software.SoftwareUpdateState; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; import javax.annotation.PostConstruct; @@ -201,20 +202,20 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl } @Override - public void onCurrentFirmwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl) { - log.debug("[{}] Current fw strategy: {}", client.getEndpoint(), newStrategy); + public void onCurrentFirmwareStrategyUpdate(LwM2mClient client, OtherConfiguration configuration) { + log.debug("[{}] Current fw strategy: {}", client.getEndpoint(), configuration.getFwUpdateStrategy()); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); - fwInfo.setFwStrategy(LwM2MFirmwareUpdateStrategy.fromStrategyFwByCode(newStrategy)); - fwInfo.setBaseUrl(newBaseUrl); + fwInfo.setFwStrategy(LwM2MFirmwareUpdateStrategy.fromStrategyFwByCode(configuration.getFwUpdateStrategy())); + fwInfo.setBaseUrl(configuration.getFwUpdateRecourse()); startFirmwareUpdateIfNeeded(client, fwInfo); } @Override - public void onCurrentSoftwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl) { - log.debug("[{}] Current sw strategy: {}", client.getEndpoint(), newStrategy); + public void onCurrentSoftwareStrategyUpdate(LwM2mClient client, OtherConfiguration configuration) { + log.debug("[{}] Current sw strategy: {}", client.getEndpoint(), configuration.getSwUpdateStrategy()); LwM2MClientOtaInfo swInfo = getOrInitSwInfo(client); - swInfo.setSwStrategy(LwM2MSoftwareUpdateStrategy.fromStrategySwByCode(newStrategy)); - swInfo.setBaseUrl(newBaseUrl); + swInfo.setSwStrategy(LwM2MSoftwareUpdateStrategy.fromStrategySwByCode(configuration.getSwUpdateStrategy())); + swInfo.setBaseUrl(configuration.getSwUpdateRecourse()); startSoftwareUpdateIfNeeded(client, swInfo); } @@ -236,8 +237,8 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl public void onCurrentFirmwareStateUpdate(LwM2mClient client, Long stateCode) { log.debug("[{}] Current fw state: {}", client.getEndpoint(), stateCode); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); - UpdateStateFw state = UpdateStateFw.fromStateFwByCode(stateCode.intValue()); - if (UpdateStateFw.DOWNLOADED.equals(state)) { + FirmwareUpdateState state = FirmwareUpdateState.fromStateFwByCode(stateCode.intValue()); + if (FirmwareUpdateState.DOWNLOADED.equals(state)) { executeFwUpdate(client); } fwInfo.setUpdateState(state); @@ -250,7 +251,7 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl public void onCurrentFirmwareResultUpdate(LwM2mClient client, Long code) { log.debug("[{}] Current fw result: {}", client.getEndpoint(), code); LwM2MClientOtaInfo fwInfo = getOrInitFwInfo(client); - UpdateResultFw result = UpdateResultFw.fromUpdateResultFwByCode(code.intValue()); + FirmwareUpdateResult result = FirmwareUpdateResult.fromUpdateResultFwByCode(code.intValue()); Optional status = this.toOtaPackageUpdateStatus(result); status.ifPresent(otaStatus -> sendStateUpdateToTelemetry(client, fwInfo, otaStatus, "Firmware Update Result: " + result.name())); @@ -324,10 +325,10 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus())) { UUID otaPackageId = new UUID(response.getOtaPackageIdMSB(), response.getOtaPackageIdLSB()); LwM2MFirmwareUpdateStrategy strategy; - if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == UpdateDeliveryFw.BOTH.code) { + if (fwInfo.getDeliveryMethod() == null || fwInfo.getDeliveryMethod() == FirmwareDeliveryMethod.BOTH.code) { strategy = fwInfo.getFwStrategy(); } else { - strategy = fwInfo.getDeliveryMethod() == UpdateDeliveryFw.PULL.code ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; + strategy = fwInfo.getDeliveryMethod() == FirmwareDeliveryMethod.PULL.code ? LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL : LwM2MFirmwareUpdateStrategy.OBJ_5_BINARY; } switch (strategy) { case OBJ_5_BINARY: @@ -405,8 +406,8 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl helper.sendParametersOnThingsboardTelemetry(result, client.getSession()); } - private static Optional toOtaPackageUpdateStatus(UpdateResultFw updateResultFw) { - switch (updateResultFw) { + private static Optional toOtaPackageUpdateStatus(FirmwareUpdateResult fwUpdateResult) { + switch (fwUpdateResult) { case INITIAL: return Optional.empty(); case UPDATE_SUCCESSFULLY: @@ -421,12 +422,12 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl case UNSUPPORTED_PROTOCOL: return Optional.of(FAILED); default: - throw new CodecException("Invalid value stateFw %s for FirmwareUpdateStatus.", updateResultFw.name()); + throw new CodecException("Invalid value stateFw %s for FirmwareUpdateStatus.", fwUpdateResult.name()); } } - private static Optional toOtaPackageUpdateStatus(UpdateStateFw updateStateFw) { - switch (updateStateFw) { + private static Optional toOtaPackageUpdateStatus(FirmwareUpdateState firmwareUpdateState) { + switch (firmwareUpdateState) { case IDLE: return Optional.empty(); case DOWNLOADING: @@ -436,7 +437,7 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl case UPDATING: return Optional.of(UPDATING); default: - throw new CodecException("Invalid value stateFw %d for FirmwareUpdateStatus.", updateStateFw); + throw new CodecException("Invalid value stateFw %d for FirmwareUpdateStatus.", firmwareUpdateState); } } @@ -444,10 +445,10 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl * FirmwareUpdateStatus { * DOWNLOADING, DOWNLOADED, VERIFIED, UPDATING, UPDATED, FAILED */ - public static Optional toSwSateResultUpdateStatus(UpdateStateSw updateStateSw, UpdateResultSw updateResultSw) { - switch (updateResultSw) { + public static Optional toSwSateResultUpdateStatus(SoftwareUpdateState softwareUpdateState, SoftwareUpdateResult softwareUpdateResult) { + switch (softwareUpdateResult) { case INITIAL: - switch (updateStateSw) { + switch (softwareUpdateState) { case INITIAL: case DOWNLOAD_STARTED: return Optional.of(DOWNLOADING); @@ -473,10 +474,8 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl case UN_INSTALL_FAILURE: return Optional.of(FAILED); default: - throw new CodecException("Invalid value stateFw %s %s for FirmwareUpdateStatus.", updateStateSw.name(), updateResultSw.name()); + throw new CodecException("Invalid value stateFw %s %s for FirmwareUpdateStatus.", softwareUpdateState.name(), softwareUpdateResult.name()); } } - - } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java index 98e2055fdc..2b503ef8a4 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MClientOtaInfo.java @@ -19,8 +19,8 @@ import lombok.Data; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.transport.lwm2m.server.ota.firmware.LwM2MFirmwareUpdateStrategy; -import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateResultFw; -import org.thingsboard.server.transport.lwm2m.server.ota.firmware.UpdateStateFw; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.FirmwareUpdateResult; +import org.thingsboard.server.transport.lwm2m.server.ota.firmware.FirmwareUpdateState; import org.thingsboard.server.transport.lwm2m.server.ota.software.LwM2MSoftwareUpdateStrategy; import java.util.Optional; @@ -47,8 +47,8 @@ public class LwM2MClientOtaInfo { //TODO: use value from device if applicable; private LwM2MFirmwareUpdateStrategy fwStrategy; private LwM2MSoftwareUpdateStrategy swStrategy; - private UpdateStateFw updateState; - private UpdateResultFw updateResult; + private FirmwareUpdateState updateState; + private FirmwareUpdateResult updateResult; private String failedPackageId; private int retryAttempts; @@ -90,7 +90,7 @@ public class LwM2MClientOtaInfo { return StringUtils.isNotEmpty(currentName) || StringUtils.isNotEmpty(currentVersion5) || StringUtils.isNotEmpty(currentVersion3); } - public void setUpdateResult(UpdateResultFw updateResult) { + public void setUpdateResult(FirmwareUpdateResult updateResult) { this.updateResult = updateResult; switch (updateResult) { case INITIAL: diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java index 644099ee9d..3ba1b1500a 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/LwM2MOtaUpdateService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.transport.lwm2m.server.ota; +import org.thingsboard.server.common.data.device.data.lwm2m.OtherConfiguration; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import java.util.Optional; @@ -31,9 +32,9 @@ public interface LwM2MOtaUpdateService { void onCurrentFirmwareNameUpdate(LwM2mClient client, String name); - void onCurrentFirmwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl); + void onCurrentFirmwareStrategyUpdate(LwM2mClient client, OtherConfiguration configuration); - void onCurrentSoftwareStrategyUpdate(LwM2mClient client, Integer newStrategy, String newBaseUrl); + void onCurrentSoftwareStrategyUpdate(LwM2mClient client, OtherConfiguration configuration); void onCurrentFirmwareVersion3Update(LwM2mClient client, String version); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateDeliveryFw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareDeliveryMethod.java similarity index 70% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateDeliveryFw.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareDeliveryMethod.java index dcb93a5283..4df6367d6f 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateDeliveryFw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareDeliveryMethod.java @@ -15,14 +15,7 @@ */ package org.thingsboard.server.transport.lwm2m.server.ota.firmware; -/** - * /** Delivery Method R - * 0: Pull only - * 1: Push only - * 2: Both: - * - In this case the LwM2M Server MAY choose the preferred mechanism for conveying the firmware image to the LwM2M Client. - */ -public enum UpdateDeliveryFw { +public enum FirmwareDeliveryMethod { PULL(0, "Pull only"), PUSH(1, "Push only"), BOTH(2, "Push or Push"); @@ -30,13 +23,13 @@ public enum UpdateDeliveryFw { public int code; public String type; - UpdateDeliveryFw(int code, String type) { + FirmwareDeliveryMethod(int code, String type) { this.code = code; this.type = type; } - public static UpdateDeliveryFw fromStateFwByType(String type) { - for (UpdateDeliveryFw to : UpdateDeliveryFw.values()) { + public static FirmwareDeliveryMethod fromStateFwByType(String type) { + for (FirmwareDeliveryMethod to : FirmwareDeliveryMethod.values()) { if (to.type.equals(type)) { return to; } @@ -44,8 +37,8 @@ public enum UpdateDeliveryFw { throw new IllegalArgumentException(String.format("Unsupported FW delivery type : %s", type)); } - public static UpdateDeliveryFw fromStateFwByCode(int code) { - for (UpdateDeliveryFw to : UpdateDeliveryFw.values()) { + public static FirmwareDeliveryMethod fromStateFwByCode(int code) { + for (FirmwareDeliveryMethod to : FirmwareDeliveryMethod.values()) { if (to.code == code) { return to; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateResultFw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareUpdateResult.java similarity index 86% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateResultFw.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareUpdateResult.java index 5a71f41bc5..cdb7d71f25 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateResultFw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareUpdateResult.java @@ -30,7 +30,7 @@ import lombok.Getter; * 8: Firmware update failed. * 9: Unsupported protocol. */ -public enum UpdateResultFw { +public enum FirmwareUpdateResult { INITIAL(0, "Initial value", false), UPDATE_SUCCESSFULLY(1, "Firmware updated successfully", false), NOT_ENOUGH(2, "Not enough flash memory for the new firmware package", false), @@ -49,14 +49,14 @@ public enum UpdateResultFw { @Getter private boolean again; - UpdateResultFw(int code, String type, boolean isAgain) { + FirmwareUpdateResult(int code, String type, boolean isAgain) { this.code = code; this.type = type; this.again = isAgain; } - public static UpdateResultFw fromUpdateResultFwByType(String type) { - for (UpdateResultFw to : UpdateResultFw.values()) { + public static FirmwareUpdateResult fromUpdateResultFwByType(String type) { + for (FirmwareUpdateResult to : FirmwareUpdateResult.values()) { if (to.type.equals(type)) { return to; } @@ -64,8 +64,8 @@ public enum UpdateResultFw { throw new IllegalArgumentException(String.format("Unsupported FW Update Result type : %s", type)); } - public static UpdateResultFw fromUpdateResultFwByCode(int code) { - for (UpdateResultFw to : UpdateResultFw.values()) { + public static FirmwareUpdateResult fromUpdateResultFwByCode(int code) { + for (FirmwareUpdateResult to : FirmwareUpdateResult.values()) { if (to.code == code) { return to; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateStateFw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareUpdateState.java similarity index 80% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateStateFw.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareUpdateState.java index 447749fd50..ddd04fbfd0 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/UpdateStateFw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/firmware/FirmwareUpdateState.java @@ -22,7 +22,7 @@ package org.thingsboard.server.transport.lwm2m.server.ota.firmware; * 2: Downloaded * 3: Updating */ -public enum UpdateStateFw { +public enum FirmwareUpdateState { IDLE(0, "Idle"), DOWNLOADING(1, "Downloading"), DOWNLOADED(2, "Downloaded"), @@ -31,13 +31,13 @@ public enum UpdateStateFw { public int code; public String type; - UpdateStateFw(int code, String type) { + FirmwareUpdateState(int code, String type) { this.code = code; this.type = type; } - public static UpdateStateFw fromStateFwByType(String type) { - for (UpdateStateFw to : UpdateStateFw.values()) { + public static FirmwareUpdateState fromStateFwByType(String type) { + for (FirmwareUpdateState to : FirmwareUpdateState.values()) { if (to.type.equals(type)) { return to; } @@ -45,8 +45,8 @@ public enum UpdateStateFw { throw new IllegalArgumentException(String.format("Unsupported FW State type : %s", type)); } - public static UpdateStateFw fromStateFwByCode(int code) { - for (UpdateStateFw to : UpdateStateFw.values()) { + public static FirmwareUpdateState fromStateFwByCode(int code) { + for (FirmwareUpdateState to : FirmwareUpdateState.values()) { if (to.code == code) { return to; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateResultSw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/SoftwareUpdateResult.java similarity index 89% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateResultSw.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/SoftwareUpdateResult.java index 22c92205d0..e99d4e4aca 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateResultSw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/SoftwareUpdateResult.java @@ -39,7 +39,7 @@ package org.thingsboard.server.transport.lwm2m.server.ota.software; * 60-200 : (for expansion, selection to be in blocks depending on new introduction of features) * This Resource MAY be reported by sending Observe operation. */ -public enum UpdateResultSw { +public enum SoftwareUpdateResult { INITIAL(0, "Initial value", false), DOWNLOADING(1, "Downloading", false), SUCCESSFULLY_INSTALLED(2, "Software successfully installed", false), @@ -58,14 +58,14 @@ public enum UpdateResultSw { public String type; public boolean isAgain; - UpdateResultSw(int code, String type, boolean isAgain) { + SoftwareUpdateResult(int code, String type, boolean isAgain) { this.code = code; this.type = type; this.isAgain = isAgain; } - public static UpdateResultSw fromUpdateResultSwByType(String type) { - for (UpdateResultSw to : UpdateResultSw.values()) { + public static SoftwareUpdateResult fromUpdateResultSwByType(String type) { + for (SoftwareUpdateResult to : SoftwareUpdateResult.values()) { if (to.type.equals(type)) { return to; } @@ -73,8 +73,8 @@ public enum UpdateResultSw { throw new IllegalArgumentException(String.format("Unsupported SW Update Result type : %s", type)); } - public static UpdateResultSw fromUpdateResultSwByCode(int code) { - for (UpdateResultSw to : UpdateResultSw.values()) { + public static SoftwareUpdateResult fromUpdateResultSwByCode(int code) { + for (SoftwareUpdateResult to : SoftwareUpdateResult.values()) { if (to.code == code) { return to; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateStateSw.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/SoftwareUpdateState.java similarity index 84% rename from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateStateSw.java rename to common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/SoftwareUpdateState.java index a5f772a11d..394d2e501a 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/UpdateStateSw.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/software/SoftwareUpdateState.java @@ -26,7 +26,7 @@ package org.thingsboard.server.transport.lwm2m.server.ota.software; * After executing the UnInstall Resource, the state changes to INITIAL. * 4: INSTALLED */ -public enum UpdateStateSw { +public enum SoftwareUpdateState { INITIAL(0, "Initial"), DOWNLOAD_STARTED(1, "DownloadStarted"), DOWNLOADED(2, "Downloaded"), @@ -36,13 +36,13 @@ public enum UpdateStateSw { public int code; public String type; - UpdateStateSw(int code, String type) { + SoftwareUpdateState(int code, String type) { this.code = code; this.type = type; } - public static UpdateStateSw fromUpdateStateSwByType(String type) { - for (UpdateStateSw to : UpdateStateSw.values()) { + public static SoftwareUpdateState fromUpdateStateSwByType(String type) { + for (SoftwareUpdateState to : SoftwareUpdateState.values()) { if (to.type.equals(type)) { return to; } @@ -50,8 +50,8 @@ public enum UpdateStateSw { throw new IllegalArgumentException(String.format("Unsupported SW State type : %s", type)); } - public static UpdateStateSw fromUpdateStateSwByCode(int code) { - for (UpdateStateSw to : UpdateStateSw.values()) { + public static SoftwareUpdateState fromUpdateStateSwByCode(int code) { + for (SoftwareUpdateState to : SoftwareUpdateState.values()) { if (to.code == code) { return to; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 3307db2fca..767c316cc6 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -40,7 +40,9 @@ import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.data.lwm2m.ObjectAttributes; +import org.thingsboard.server.common.data.device.data.lwm2m.OtherConfiguration; import org.thingsboard.server.common.data.device.data.lwm2m.TelemetryMappingConfiguration; import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.ota.OtaPackageUtil; @@ -130,16 +132,13 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl private final LwM2mTransportContext context; private final LwM2MAttributesService attributesService; private final LwM2MOtaUpdateService otaService; - public final LwM2MTransportServerConfig config; + private final LwM2MTransportServerConfig config; private final LwM2MTelemetryLogService logService; - public final OtaPackageDataCache otaPackageDataCache; - public final LwM2mTransportServerHelper helper; + private final LwM2mTransportServerHelper helper; private final TbLwM2MDtlsSessionStore sessionStore; - public final LwM2mClientContext clientContext; + private final LwM2mClientContext clientContext; private final LwM2MRpcRequestHandler rpcHandler; - public final LwM2mDownlinkMsgHandler defaultLwM2MDownlinkMsgHandler; - - public final Map firmwareUpdateState; + private final LwM2mDownlinkMsgHandler defaultLwM2MDownlinkMsgHandler; public DefaultLwM2MUplinkMsgHandler(TransportService transportService, LwM2MTransportServerConfig config, @@ -150,7 +149,6 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl @Lazy LwM2MAttributesService attributesService, @Lazy LwM2MRpcRequestHandler rpcHandler, @Lazy LwM2mDownlinkMsgHandler defaultLwM2MDownlinkMsgHandler, - OtaPackageDataCache otaPackageDataCache, LwM2mTransportContext context, TbLwM2MDtlsSessionStore sessionStore) { this.transportService = transportService; this.attributesService = attributesService; @@ -161,9 +159,7 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl this.logService = logService; this.rpcHandler = rpcHandler; this.defaultLwM2MDownlinkMsgHandler = defaultLwM2MDownlinkMsgHandler; - this.otaPackageDataCache = otaPackageDataCache; this.context = context; - this.firmwareUpdateState = new ConcurrentHashMap<>(); this.sessionStore = sessionStore; } @@ -781,27 +777,19 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl } } - // # 7.1 // update value in fwInfo - if (!newProfile.getClientLwM2mSettings().getFwUpdateStrategy().equals(oldProfile.getClientLwM2mSettings().getFwUpdateStrategy()) - || (LwM2MFirmwareUpdateStrategy.OBJ_5_TEMP_URL.code == newProfile.getClientLwM2mSettings().getFwUpdateStrategy() && - !newProfile.getClientLwM2mSettings().getFwUpdateRecourse().equals(oldProfile.getClientLwM2mSettings().getFwUpdateRecourse()))) { - clients.forEach(lwM2MClient -> { - otaService.onCurrentFirmwareStrategyUpdate(lwM2MClient, - newProfile.getClientLwM2mSettings().getFwUpdateStrategy(), - newProfile.getClientLwM2mSettings().getFwUpdateRecourse()); - }); + OtherConfiguration newLwM2mSettings = newProfile.getClientLwM2mSettings(); + OtherConfiguration oldLwM2mSettings = oldProfile.getClientLwM2mSettings(); + if (!newLwM2mSettings.getFwUpdateStrategy().equals(oldLwM2mSettings.getFwUpdateStrategy()) + || (StringUtils.isNotEmpty(newLwM2mSettings.getFwUpdateRecourse()) && + !newLwM2mSettings.getFwUpdateRecourse().equals(oldLwM2mSettings.getFwUpdateRecourse()))) { + clients.forEach(lwM2MClient -> otaService.onCurrentFirmwareStrategyUpdate(lwM2MClient, newLwM2mSettings)); } - //# 7.2 // update value in swInfo - if (!newProfile.getClientLwM2mSettings().getSwUpdateStrategy().equals(oldProfile.getClientLwM2mSettings().getSwUpdateStrategy()) - || (LwM2MSoftwareUpdateStrategy.TEMP_URL.code == newProfile.getClientLwM2mSettings().getSwUpdateStrategy() && - !newProfile.getClientLwM2mSettings().getSwUpdateRecourse().equals(oldProfile.getClientLwM2mSettings().getSwUpdateRecourse()))) { - clients.forEach(lwM2MClient -> { - otaService.onCurrentSoftwareStrategyUpdate(lwM2MClient, - newProfile.getClientLwM2mSettings().getFwUpdateStrategy(), - newProfile.getClientLwM2mSettings().getFwUpdateRecourse()); - }); + if (!newLwM2mSettings.getSwUpdateStrategy().equals(oldLwM2mSettings.getSwUpdateStrategy()) + || (StringUtils.isNotEmpty(newLwM2mSettings.getSwUpdateRecourse()) && + !newLwM2mSettings.getSwUpdateRecourse().equals(oldLwM2mSettings.getSwUpdateRecourse()))) { + clients.forEach(lwM2MClient -> otaService.onCurrentSoftwareStrategyUpdate(lwM2MClient, newLwM2mSettings)); } } }