From f81413112c2858e44dc308c056c38f8fbc5ed94f Mon Sep 17 00:00:00 2001 From: Artem Babak Date: Wed, 26 May 2021 21:02:35 +0300 Subject: [PATCH 01/76] Edge: menu order changes --- ui-ngx/src/app/core/services/menu.service.ts | 32 +++++++++---------- .../pages/customer/customer-routing.module.ts | 2 +- .../customers-table-config.resolver.ts | 2 +- .../dashboards-table-config.resolver.ts | 2 +- .../home/pages/edge/edge-routing.module.ts | 26 +++++++++++++-- .../pages/edge/edges-table-config.resolver.ts | 2 +- .../rulechain/rule-node-details.component.ts | 2 +- .../rulechain/rulechain-page.component.ts | 2 +- .../rulechains-table-config.resolver.ts | 6 ++-- .../pages/rulechain/rulenode.component.ts | 2 +- 10 files changed, 49 insertions(+), 29 deletions(-) diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 07a6d8f237..10dedd4fc7 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -291,26 +291,26 @@ export class MenuService { ); if (authState.edgesSupportEnabled) { sections.push( + { + id: guid(), + name: 'edge.edge-instances', + type: 'link', + path: '/edgeInstances', + icon: 'router' + }, { id: guid(), name: 'edge.management', type: 'toggle', - path: '/edges', - height: '80px', - icon: 'router', + path: '/edgeManagement', + height: '40px', + icon: 'settings_input_antenna', pages: [ - { - id: guid(), - name: 'edge.edge-instances', - type: 'link', - path: '/edges', - icon: 'router' - }, { id: guid(), name: 'edge.rulechain-templates', type: 'link', - path: '/edges/ruleChains', + path: '/edgeManagement/ruleChains', icon: 'settings_ethernet' } ] @@ -448,12 +448,12 @@ export class MenuService { { name: 'edge.edge-instances', icon: 'router', - path: '/edges' + path: '/edgeInstances' }, { name: 'edge.rulechain-templates', icon: 'settings_ethernet', - path: '/edges/ruleChains' + path: '/edgeManagement/ruleChains' } ] } @@ -548,7 +548,7 @@ export class MenuService { id: guid(), name: 'edge.edge-instances', type: 'link', - path: '/edges', + path: '/edgeInstances', icon: 'router' } ); @@ -606,8 +606,8 @@ export class MenuService { places: [ { name: 'edge.edge-instances', - icon: 'router', - path: '/edges' + icon: 'settings_input_antenna', + path: '/edgeInstances' } ] } diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts b/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts index f5f3ee05fa..2ed306c2d8 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/customer/customer-routing.module.ts @@ -98,7 +98,7 @@ const routes: Routes = [ } }, { - path: ':customerId/edges', + path: ':customerId/edgeInstances', component: EntitiesTableComponent, data: { auth: [Authority.TENANT_ADMIN], diff --git a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts index 8ac9e64390..5a6e03b3d7 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts @@ -169,7 +169,7 @@ export class CustomersTableConfigResolver implements Resolve): boolean { diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts index c572fa958f..a4c17e75ac 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts @@ -362,7 +362,7 @@ export class DashboardsTableConfigResolver implements Resolve) { diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts index 00cd38b9c4..d5a3fcc8a0 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-details.component.ts @@ -152,7 +152,7 @@ export class RuleNodeDetailsComponent extends PageComponent implements OnInit, O } if (this.ruleNode.targetRuleChainId) { if (this.ruleChainType === RuleChainType.EDGE) { - this.router.navigateByUrl(`/edges/ruleChains/${this.ruleNode.targetRuleChainId}`); + this.router.navigateByUrl(`/edgeManagement/ruleChains/${this.ruleNode.targetRuleChainId}`); } else { this.router.navigateByUrl(`/ruleChains/${this.ruleNode.targetRuleChainId}`); } diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts index 54102ffbcd..ea9cc54747 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.ts @@ -1289,7 +1289,7 @@ export class RuleChainPageComponent extends PageComponent if (this.ruleChainType !== RuleChainType.EDGE) { this.router.navigateByUrl(`ruleChains/${this.ruleChain.id.id}`); } else { - this.router.navigateByUrl(`edges/ruleChains/${this.ruleChain.id.id}`); + this.router.navigateByUrl(`edgeManagement/ruleChains/${this.ruleChain.id.id}`); } } else { this.createRuleChainModel(); diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts index 37a21cab2e..ffc05052fe 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts @@ -285,7 +285,7 @@ export class RuleChainsTableConfigResolver implements Resolve Date: Thu, 27 May 2021 09:39:53 +0300 Subject: [PATCH 02/76] Minor code cleanup --- .../src/app/modules/home/pages/edge/edge-routing.module.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts b/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts index bd96b953a8..4ffa51a6bd 100644 --- a/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/edge/edge-routing.module.ts @@ -39,7 +39,6 @@ import { RuleNodeComponentsResolver, TooltipsterResolver } from '@home/pages/rulechain/rulechain-routing.module'; -import { CustomersTableConfigResolver } from '@home/pages/customer/customers-table-config.resolver'; const routes: Routes = [ { @@ -56,7 +55,7 @@ const routes: Routes = [ component: EntitiesTableComponent, data: { auth: [Authority.TENANT_ADMIN, Authority.CUSTOMER_USER], - edgesType: 'tenant', + edgesType: 'tenant' }, resolve: { entitiesTableConfig: EdgesTableConfigResolver @@ -272,8 +271,7 @@ const routes: Routes = [ ] } ] - } - ]; + }]; @NgModule({ imports: [RouterModule.forChild(routes)], From 0c886656543987f27d59c2c053c11677de147a8b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 25 May 2021 19:44:34 +0300 Subject: [PATCH 03/76] js-executor: scriptMap refactored from Object to the Map() --- .../api/jsInvokeMessageProcessor.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 2964291a30..a760e21e47 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -34,10 +34,9 @@ const slowQueryLogBody = config.get('script.slow_query_log_body') === 'true'; const {performance} = require('perf_hooks'); function JsInvokeMessageProcessor(producer) { - console.log("Producer:", producer); this.producer = producer; this.executor = new JsExecutor(useSandbox); - this.scriptMap = {}; + this.scriptMap = new Map(); this.scriptIds = []; this.executedScriptsCounter = 0; } @@ -157,12 +156,12 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) { var scriptId = getScriptId(releaseRequest); logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); - if (this.scriptMap[scriptId]) { + if (this.scriptMap.has(scriptId)) { var index = this.scriptIds.indexOf(scriptId); if (index > -1) { this.scriptIds.splice(index, 1); } - delete this.scriptMap[scriptId]; + this.scriptMap.delete(scriptId); } var releaseResponse = createReleaseResponse(scriptId, true); logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId); @@ -189,8 +188,8 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { var self = this; return new Promise(function(resolve, reject) { - if (self.scriptMap[scriptId]) { - resolve(self.scriptMap[scriptId]); + if (self.scriptMap.has(scriptId)) { + resolve(self.scriptMap.get(scriptId)); } else { self.executor.compileScript(scriptBody).then( (script) => { @@ -206,16 +205,17 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scrip } JsInvokeMessageProcessor.prototype.cacheScript = function(scriptId, script) { - if (!this.scriptMap[scriptId]) { + if (!this.scriptMap.has(scriptId)) { this.scriptIds.push(scriptId); while (this.scriptIds.length > maxActiveScripts) { logger.info('Active scripts count [%s] exceeds maximum limit [%s]', this.scriptIds.length, maxActiveScripts); const prevScriptId = this.scriptIds.shift(); logger.info('Removing active script with id [%s]', prevScriptId); - delete this.scriptMap[prevScriptId]; + this.scriptMap.delete(prevScriptId); } } - this.scriptMap[scriptId] = script; + this.scriptMap.set(scriptId, script); + logger.info("scriptMap size is [%s]", this.scriptMap.size); } function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) { From d729d9ee959963b570aeeaf27787cdd6b6f44bdf Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 27 May 2021 14:50:42 +0300 Subject: [PATCH 04/76] js-executor: instrumentation event for producer and consumer to define the exact flow how to Kafka works without batches (for debug only) --- msa/js-executor/queue/kafkaTemplate.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 2f4fd8751d..1fd21185d0 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -114,6 +114,30 @@ function KafkaProducer() { consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); producer = kafkaClient.producer(); + + const { CONNECT } = producer.events; + const removeListenerC = producer.on(CONNECT, e => logger.info(`producer CONNECT`)); + const { DISCONNECT } = producer.events; + const removeListenerD = producer.on(DISCONNECT, e => logger.info(`producer DISCONNECT`)); + const { REQUEST } = producer.events; + const removeListenerR = producer.on(REQUEST, e => logger.info(`producer REQUEST ${e.payload.broker}`)); + const { REQUEST_TIMEOUT } = producer.events; + const removeListenerRT = producer.on(REQUEST_TIMEOUT, e => logger.info(`producer REQUEST_TIMEOUT ${e.payload.broker}`)); + const { REQUEST_QUEUE_SIZE } = producer.events; + const removeListenerRQS = producer.on(REQUEST_QUEUE_SIZE, e => logger.info(`producer REQUEST_QUEUE_SIZE ${e.payload.broker} size ${e.queueSize}`)); + + const removeListeners = {} + const { FETCH_START } = consumer.events; + removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); + const { FETCH } = consumer.events; + removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); + const { START_BATCH_PROCESS } = consumer.events; + removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { END_BATCH_PROCESS } = consumer.events; + removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { COMMIT_OFFSETS } = consumer.events; + removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); + const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); From 35e2ff99c3d3f7a488fdc4d3fb3375daa06d41f9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 27 May 2021 14:55:10 +0300 Subject: [PATCH 05/76] js-executor: send messages as batch --- .../api/jsInvokeMessageProcessor.js | 25 ++++---- msa/js-executor/queue/kafkaTemplate.js | 63 ++++++++++++++----- 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index a760e21e47..867dd82b39 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -172,17 +172,20 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT var tStartSending = performance.now(); var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); - this.producer.send(responseTopic, scriptId, rawResponse, headers).then( - () => { - logger.debug('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); - }, - (err) => { - if (err) { - logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); - logger.error(err.stack); - } - } - ); + logger.debug('[%s] Sending response to queue, scriptId: [%s]', requestId, scriptId); + this.producer.send(responseTopic, scriptId, rawResponse, headers); +//TODO put error msg for other queues implementation except Kafka +// .then( +// () => { +// logger.info('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); +// }, +// (err) => { +// if (err) { +// logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); +// logger.error(err.stack); +// } +// } +// ); } JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 1fd21185d0..da46a45301 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -33,24 +33,54 @@ let producer; const configEntries = []; +let topicMessages = []; +let loopSend; + function KafkaProducer() { - this.send = async (responseTopic, scriptId, rawResponse, headers) => { - return producer.send( - { - topic: responseTopic, - acks: acks, - compression: compressionType, - messages: [ - { - key: scriptId, - value: rawResponse, - headers: headers.data - } - ] - }); + this.send = (responseTopic, scriptId, rawResponse, headers) => { + logger.debug('Pending queue response, scriptId: [%s]', scriptId); + const message = { + topic: responseTopic, + messages: [{ + key: scriptId, + value: rawResponse, + headers: headers.data + }] + }; + + topicMessages.push(message); + return {}; } } +function sendLoopFunction() { + loopSend = setInterval(sendProducerMsg, 200); +} + +function sendProducerMsg() { + if (topicMessages.length > 0) { + logger.info('sendProducerMsg from queue response, lenght: [%s]', topicMessages.length ); + const messagesToSend = topicMessages; + topicMessages = []; + producer.sendBatch({ + topicMessages: messagesToSend, + acks: acks, + compression: compressionType + }).then( + () => { + logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); + }, + (err) => { + if (err) { + logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); + topicMessages = messagesToSend.concat(topicMessages); + logger.error(err.stack); + } + } + ); + } +} + (async () => { try { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); @@ -141,6 +171,7 @@ function KafkaProducer() { const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); + sendLoopFunction(); await consumer.subscribe({topic: requestTopic}); logger.info('Started ThingsBoard JavaScript Executor Microservice.'); @@ -221,6 +252,10 @@ async function disconnectProducer() { var _producer = producer; producer = null; try { + logger.info('Stopping loop...'); + //TODO: send handle msg + clearInterval(loopSend); + sendProducerMsg(); await _producer.disconnect(); logger.info('Kafka Producer stopped.'); } catch (e) { From 0970ce65b448f622056c070831aa1625f1e81f36 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 27 May 2021 18:45:29 +0300 Subject: [PATCH 06/76] js-executor: maxBatchSize --- msa/js-executor/queue/kafkaTemplate.js | 88 ++++++++++++++++---------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index da46a45301..a061215b28 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -26,6 +26,9 @@ const acks = Number(config.get('kafka.acks')); const requestTimeout = Number(config.get('kafka.requestTimeout')); const compressionType = (config.get('kafka.requestTimeout') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; +const linger = 5; //milliseconds //TODO move to the config +const maxBatchSize = 10; //max messages in batch //TODO move to the config + let kafkaClient; let kafkaAdmin; let consumer; @@ -33,8 +36,8 @@ let producer; const configEntries = []; -let topicMessages = []; -let loopSend; +let batchMessages = []; +let sendLoopInstance; function KafkaProducer() { this.send = (responseTopic, scriptId, rawResponse, headers) => { @@ -48,36 +51,51 @@ function KafkaProducer() { }] }; - topicMessages.push(message); + pushMessageToSendLater(message); return {}; } } -function sendLoopFunction() { - loopSend = setInterval(sendProducerMsg, 200); +function pushMessageToSendLater(message) { + batchMessages.push(message); + if (batchMessages.length >= maxBatchSize) { + sendMessagesAsBatch(); + sendLoopWithLinger(); //reset loop function and reschedule new linger + } +} + +function sendLoopWithLinger() { + if (sendLoopInstance) { + logger.debug("Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); + clearInterval(sendLoopInstance); + } else { + logger.debug("Starting new send loop with linger [%s]", linger) + } + sendLoopInstance = setInterval(sendMessagesAsBatch, linger); } -function sendProducerMsg() { - if (topicMessages.length > 0) { - logger.info('sendProducerMsg from queue response, lenght: [%s]', topicMessages.length ); - const messagesToSend = topicMessages; - topicMessages = []; +function sendMessagesAsBatch() { + if (batchMessages.length > 0) { + logger.info('sendMessagesAsBatch, lenght: [%s]', batchMessages.length ); + const messagesToSend = batchMessages; + batchMessages = []; producer.sendBatch({ topicMessages: messagesToSend, acks: acks, compression: compressionType }).then( - () => { - logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); - }, - (err) => { - if (err) { - logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); - topicMessages = messagesToSend.concat(topicMessages); - logger.error(err.stack); - } - } - ); + () => { + logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); + }, + (err) => { + logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); + batchMessages = messagesToSend.concat(batchMessages); + logger.error(err.stack); + } + ); + + } else { + //logger.debug("nothing to send"); } } @@ -156,22 +174,22 @@ function sendProducerMsg() { const { REQUEST_QUEUE_SIZE } = producer.events; const removeListenerRQS = producer.on(REQUEST_QUEUE_SIZE, e => logger.info(`producer REQUEST_QUEUE_SIZE ${e.payload.broker} size ${e.queueSize}`)); - const removeListeners = {} - const { FETCH_START } = consumer.events; - removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); - const { FETCH } = consumer.events; - removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); - const { START_BATCH_PROCESS } = consumer.events; - removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); - const { END_BATCH_PROCESS } = consumer.events; - removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); - const { COMMIT_OFFSETS } = consumer.events; - removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); +// const removeListeners = {} +// const { FETCH_START } = consumer.events; +// removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); +// const { FETCH } = consumer.events; +// removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); +// const { START_BATCH_PROCESS } = consumer.events; +// removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); +// const { END_BATCH_PROCESS } = consumer.events; +// removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); +// const { COMMIT_OFFSETS } = consumer.events; +// removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); - sendLoopFunction(); + sendLoopWithLinger(); await consumer.subscribe({topic: requestTopic}); logger.info('Started ThingsBoard JavaScript Executor Microservice.'); @@ -254,8 +272,8 @@ async function disconnectProducer() { try { logger.info('Stopping loop...'); //TODO: send handle msg - clearInterval(loopSend); - sendProducerMsg(); + clearInterval(sendLoopInstance); + sendMessagesAsBatch(); await _producer.disconnect(); logger.info('Kafka Producer stopped.'); } catch (e) { From 93bea7020584343511b1a561bb861fd010df037e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 28 May 2021 09:41:35 +0300 Subject: [PATCH 07/76] js-executor: added parameters for producer TB_KAFKA_BATCH_SIZE and TB_KAFKA_LINGER_MS; added print stats frequency SCRIPT_STAT_PRINT_FREQUENCY --- .../api/jsInvokeMessageProcessor.js | 17 +++++--- .../config/custom-environment-variables.yml | 3 ++ msa/js-executor/config/default.yml | 7 ++- msa/js-executor/queue/kafkaTemplate.js | 43 +++++++++++-------- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 867dd82b39..dec935c8ad 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -25,6 +25,7 @@ const config = require('config'), Utils = require('./utils'), JsExecutor = require('./jsExecutor'); +const statFrequency = Number(config.get('script.stat_print_frequency')); const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency')); const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); @@ -39,6 +40,7 @@ function JsInvokeMessageProcessor(producer) { this.scriptMap = new Map(); this.scriptIds = []; this.executedScriptsCounter = 0; + this.lastStatTime = performance.now(); } JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { @@ -118,11 +120,16 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re var scriptId = getScriptId(invokeRequest); logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; - if ( this.executedScriptsCounter >= scriptBodyTraceFrequency ) { - this.executedScriptsCounter = 0; - if (logger.levels[logger.level] >= logger.levels['debug']) { - logger.debug('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody); - } + if (this.executedScriptsCounter % statFrequency == 0) { + var nowMs = performance.now(); + var msSinceLastStat = nowMs - this.lastStatTime; + var requestsPerSec = statFrequency / msSinceLastStat * 1000; //msSinceLastStat can't be zero in the real world + this.lastStatTime = nowMs; + logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s]', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec); + } + + if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) { + logger.info('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody); } this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then( (script) => { diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 5e2bfe8342..c5eb15e8b9 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -26,6 +26,8 @@ kafka: servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge + batch_size: "${TB_KAFKA_BATCH_SIZE:128}" # for producer + linger_ms: "${TB_KAFKA_LINGER_MS:1}" # for producer requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" @@ -70,6 +72,7 @@ logger: script: use_sandbox: "SCRIPT_USE_SANDBOX" + stat_print_frequency: "SCRIPT_STAT_PRINT_FREQUENCY" script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY" max_active_scripts: "MAX_ACTIVE_SCRIPTS" slow_query_log_ms: "SLOW_QUERY_LOG_MS" #1.123456 diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index cdf5b35421..5a2c47a675 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -26,6 +26,8 @@ kafka: servers: "localhost:9092" replication_factor: "1" acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge + batch_size: "128" # for producer + linger_ms: "1" # for producer requestTimeout: "30000" # The default value in kafkajs is: 30000 compression: "gzip" # gzip or uncompressed topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1" @@ -59,7 +61,8 @@ logger: script: use_sandbox: "true" - script_body_trace_frequency: "1000" + script_body_trace_frequency: "10000" + stat_print_frequency: "10000" max_active_scripts: "1000" - slow_query_log_ms: "1.000000" #millis + slow_query_log_ms: "5.000000" #millis slow_query_log_body: "false" diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index a061215b28..79d87c17b4 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -23,12 +23,11 @@ const replicationFactor = Number(config.get('kafka.replication_factor')); const topicProperties = config.get('kafka.topic_properties'); const kafkaClientId = config.get('kafka.client_id'); const acks = Number(config.get('kafka.acks')); +const maxBatchSize = Number(config.get('kafka.batch_size')); +const linger = Number(config.get('kafka.linger_ms')); const requestTimeout = Number(config.get('kafka.requestTimeout')); const compressionType = (config.get('kafka.requestTimeout') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; -const linger = 5; //milliseconds //TODO move to the config -const maxBatchSize = 10; //max messages in batch //TODO move to the config - let kafkaClient; let kafkaAdmin; let consumer; @@ -76,7 +75,9 @@ function sendLoopWithLinger() { function sendMessagesAsBatch() { if (batchMessages.length > 0) { - logger.info('sendMessagesAsBatch, lenght: [%s]', batchMessages.length ); + if (batchMessages.length > 1) { + logger.info('sendMessagesAsBatch, length: [%s]', batchMessages.length); + } const messagesToSend = batchMessages; batchMessages = []; producer.sendBatch({ @@ -85,17 +86,15 @@ function sendMessagesAsBatch() { compression: compressionType }).then( () => { - logger.info('Response sent to kafka, length: [%s]', messagesToSend.length ); + logger.debug('Response sent to kafka, length: [%s]', messagesToSend.length); }, (err) => { - logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length ); + logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); batchMessages = messagesToSend.concat(batchMessages); logger.error(err.stack); } ); - } else { - //logger.debug("nothing to send"); } } @@ -163,6 +162,8 @@ function sendMessagesAsBatch() { consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); producer = kafkaClient.producer(); +/* + //producer event instrumentation to debug const { CONNECT } = producer.events; const removeListenerC = producer.on(CONNECT, e => logger.info(`producer CONNECT`)); const { DISCONNECT } = producer.events; @@ -173,18 +174,22 @@ function sendMessagesAsBatch() { const removeListenerRT = producer.on(REQUEST_TIMEOUT, e => logger.info(`producer REQUEST_TIMEOUT ${e.payload.broker}`)); const { REQUEST_QUEUE_SIZE } = producer.events; const removeListenerRQS = producer.on(REQUEST_QUEUE_SIZE, e => logger.info(`producer REQUEST_QUEUE_SIZE ${e.payload.broker} size ${e.queueSize}`)); +*/ -// const removeListeners = {} -// const { FETCH_START } = consumer.events; -// removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); -// const { FETCH } = consumer.events; -// removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); -// const { START_BATCH_PROCESS } = consumer.events; -// removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); -// const { END_BATCH_PROCESS } = consumer.events; -// removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); -// const { COMMIT_OFFSETS } = consumer.events; -// removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); +/* + //consumer event instrumentation to debug + const removeListeners = {} + const { FETCH_START } = consumer.events; + removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`)); + const { FETCH } = consumer.events; + removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`)); + const { START_BATCH_PROCESS } = consumer.events; + removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { END_BATCH_PROCESS } = consumer.events; + removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`)); + const { COMMIT_OFFSETS } = consumer.events; + removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`)); +*/ const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); From 2cba1e2f16e9a1a438d2e4bf7af0afa8968fbae5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 28 May 2021 18:07:31 +0300 Subject: [PATCH 08/76] js-executor reduced log severity to debug --- msa/js-executor/queue/kafkaTemplate.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 79d87c17b4..f467d0813b 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -75,9 +75,7 @@ function sendLoopWithLinger() { function sendMessagesAsBatch() { if (batchMessages.length > 0) { - if (batchMessages.length > 1) { - logger.info('sendMessagesAsBatch, length: [%s]', batchMessages.length); - } + logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); const messagesToSend = batchMessages; batchMessages = []; producer.sendBatch({ From a4e28ad94533610c4de0f855bad75dcd4460ac74 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 1 Jun 2021 16:00:32 +0300 Subject: [PATCH 09/76] js-executor fixed promises for each message for Kafka batches --- .../api/jsInvokeMessageProcessor.js | 24 +++++++------- msa/js-executor/queue/kafkaTemplate.js | 33 ++++++++++++------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index dec935c8ad..1aabd216d2 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -180,19 +180,17 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); logger.debug('[%s] Sending response to queue, scriptId: [%s]', requestId, scriptId); - this.producer.send(responseTopic, scriptId, rawResponse, headers); -//TODO put error msg for other queues implementation except Kafka -// .then( -// () => { -// logger.info('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); -// }, -// (err) => { -// if (err) { -// logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); -// logger.error(err.stack); -// } -// } -// ); + this.producer.send(responseTopic, scriptId, rawResponse, headers).then( + () => { + logger.debug('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId); + }, + (err) => { + if (err) { + logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); + logger.error(err.stack); + } + } + ); } JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index f467d0813b..1fac350976 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -36,11 +36,12 @@ let producer; const configEntries = []; let batchMessages = []; +let batchResolvers = []; let sendLoopInstance; function KafkaProducer() { this.send = (responseTopic, scriptId, rawResponse, headers) => { - logger.debug('Pending queue response, scriptId: [%s]', scriptId); + logger.debug('Pending queue response, scriptId: [%s]', scriptId); const message = { topic: responseTopic, messages: [{ @@ -50,17 +51,22 @@ function KafkaProducer() { }] }; - pushMessageToSendLater(message); - return {}; + return pushMessageToSendLater(message); } } function pushMessageToSendLater(message) { + let resolver; + const promise = new Promise((resolve, reject) => { + resolver = resolve; + }); batchMessages.push(message); + batchResolvers.push(resolver); if (batchMessages.length >= maxBatchSize) { sendMessagesAsBatch(); sendLoopWithLinger(); //reset loop function and reschedule new linger } + return promise; } function sendLoopWithLinger() { @@ -77,22 +83,27 @@ function sendMessagesAsBatch() { if (batchMessages.length > 0) { logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); const messagesToSend = batchMessages; + const resolvers = batchResolvers; batchMessages = []; + batchResolvers = []; producer.sendBatch({ topicMessages: messagesToSend, acks: acks, compression: compressionType }).then( - () => { - logger.debug('Response sent to kafka, length: [%s]', messagesToSend.length); - }, - (err) => { - logger.error('Failed to send kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); - batchMessages = messagesToSend.concat(batchMessages); - logger.error(err.stack); + () => { + logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); + for (let i = 0; i < promisesToSend.length; i++) { + resolvers[i](); } + }, + (err) => { + logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); + logger.error(err.stack); + batchMessages = messagesToSend.concat(batchMessages); + batchResolvers = resolvers.concat(batchResolvers); //promises will never be rejected. Will retry forever + } ); - } } From 1d9fc4a322f8067b6e35bc96e8ff28eb13f2ca8e Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 1 Jun 2021 18:01:21 +0300 Subject: [PATCH 10/76] js-executor: typo fix --- msa/js-executor/queue/kafkaTemplate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 1fac350976..ad243aacdd 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -93,7 +93,7 @@ function sendMessagesAsBatch() { }).then( () => { logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); - for (let i = 0; i < promisesToSend.length; i++) { + for (let i = 0; i < resolvers.length; i++) { resolvers[i](); } }, From 41391dbef8bbfbf1c0f56165dd41277dc45d23fe Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 1 Jun 2021 18:06:10 +0300 Subject: [PATCH 11/76] js-executor: code format --- .../api/jsInvokeMessageProcessor.js | 56 +++++++++---------- msa/js-executor/queue/kafkaTemplate.js | 46 +++++++-------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 1aabd216d2..bef70936ec 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -43,7 +43,7 @@ function JsInvokeMessageProcessor(producer) { this.lastStatTime = performance.now(); } -JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { +JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function (message) { var tStart = performance.now(); let requestId; let responseTopic; @@ -78,13 +78,13 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var tFinish = performance.now(); var tTook = tFinish - tStart; - if ( tTook > slowQueryLogMs ) { + if (tTook > slowQueryLogMs) { let functionName; if (request.invokeRequest) { try { buf = Buffer.from(request.invokeRequest['functionName']); functionName = buf.toString('utf8'); - } catch (err){ + } catch (err) { logger.error('[%s] Failed to read functionName from message header: %s', requestId, err.message); logger.error(err.stack); } @@ -97,7 +97,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { } -JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, headers, compileRequest) { +JsInvokeMessageProcessor.prototype.processCompileRequest = function (requestId, responseTopic, headers, compileRequest) { var scriptId = getScriptId(compileRequest); logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId); @@ -116,7 +116,7 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r ); } -JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, headers, invokeRequest) { +JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, responseTopic, headers, invokeRequest) { var scriptId = getScriptId(invokeRequest); logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; @@ -160,7 +160,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re ); } -JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) { +JsInvokeMessageProcessor.prototype.processReleaseRequest = function (requestId, responseTopic, headers, releaseRequest) { var scriptId = getScriptId(releaseRequest); logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); if (this.scriptMap.has(scriptId)) { @@ -193,9 +193,9 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT ); } -JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { +JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scriptBody) { var self = this; - return new Promise(function(resolve, reject) { + return new Promise(function (resolve, reject) { if (self.scriptMap.has(scriptId)) { resolve(self.scriptMap.get(scriptId)); } else { @@ -212,7 +212,7 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scrip }); } -JsInvokeMessageProcessor.prototype.cacheScript = function(scriptId, script) { +JsInvokeMessageProcessor.prototype.cacheScript = function (scriptId, script) { if (!this.scriptMap.has(scriptId)) { this.scriptIds.push(scriptId); while (this.scriptIds.length > maxActiveScripts) { @@ -229,40 +229,40 @@ JsInvokeMessageProcessor.prototype.cacheScript = function(scriptId, script) { function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) { const requestIdBits = Utils.UUIDToBits(requestId); return { - requestIdMSB: requestIdBits[0], - requestIdLSB: requestIdBits[1], - compileResponse: compileResponse, - invokeResponse: invokeResponse, - releaseResponse: releaseResponse + requestIdMSB: requestIdBits[0], + requestIdLSB: requestIdBits[1], + compileResponse: compileResponse, + invokeResponse: invokeResponse, + releaseResponse: releaseResponse }; } function createCompileResponse(scriptId, success, errorCode, err) { const scriptIdBits = Utils.UUIDToBits(scriptId); - return { - errorCode: errorCode, - success: success, - errorDetails: parseJsErrorDetails(err), - scriptIdMSB: scriptIdBits[0], - scriptIdLSB: scriptIdBits[1] + return { + errorCode: errorCode, + success: success, + errorDetails: parseJsErrorDetails(err), + scriptIdMSB: scriptIdBits[0], + scriptIdLSB: scriptIdBits[1] }; } function createInvokeResponse(result, success, errorCode, err) { - return { - errorCode: errorCode, - success: success, - errorDetails: parseJsErrorDetails(err), - result: result + return { + errorCode: errorCode, + success: success, + errorDetails: parseJsErrorDetails(err), + result: result }; } function createReleaseResponse(scriptId, success) { const scriptIdBits = Utils.UUIDToBits(scriptId); return { - success: success, - scriptIdMSB: scriptIdBits[0], - scriptIdLSB: scriptIdBits[1] + success: success, + scriptIdMSB: scriptIdBits[0], + scriptIdLSB: scriptIdBits[1] }; } diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index ad243aacdd..7fce429d23 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -45,9 +45,9 @@ function KafkaProducer() { const message = { topic: responseTopic, messages: [{ - key: scriptId, - value: rawResponse, - headers: headers.data + key: scriptId, + value: rawResponse, + headers: headers.data }] }; @@ -70,28 +70,28 @@ function pushMessageToSendLater(message) { } function sendLoopWithLinger() { - if (sendLoopInstance) { + if (sendLoopInstance) { logger.debug("Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); clearInterval(sendLoopInstance); - } else { + } else { logger.debug("Starting new send loop with linger [%s]", linger) - } - sendLoopInstance = setInterval(sendMessagesAsBatch, linger); + } + sendLoopInstance = setInterval(sendMessagesAsBatch, linger); } function sendMessagesAsBatch() { - if (batchMessages.length > 0) { - logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); - const messagesToSend = batchMessages; - const resolvers = batchResolvers; - batchMessages = []; - batchResolvers = []; - producer.sendBatch({ - topicMessages: messagesToSend, - acks: acks, - compression: compressionType - }).then( - () => { + if (batchMessages.length > 0) { + logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); + const messagesToSend = batchMessages; + const resolvers = batchResolvers; + batchMessages = []; + batchResolvers = []; + producer.sendBatch({ + topicMessages: messagesToSend, + acks: acks, + compression: compressionType + }).then( + () => { logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); for (let i = 0; i < resolvers.length; i++) { resolvers[i](); @@ -103,8 +103,8 @@ function sendMessagesAsBatch() { batchMessages = messagesToSend.concat(batchMessages); batchResolvers = resolvers.concat(batchResolvers); //promises will never be rejected. Will retry forever } - ); - } + ); + } } (async () => { @@ -120,8 +120,8 @@ function sendMessagesAsBatch() { let kafkaConfig = { brokers: kafkaBootstrapServers.split(','), - logLevel: logLevel.INFO, - logCreator: KafkaJsWinstonLogCreator + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator }; if (kafkaClientId) { From ab5f1b5b639236fb7528ff9544425e2310c33dbb Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 2 Jun 2021 10:41:24 +0300 Subject: [PATCH 12/76] js-executor: ScriptMap optimize work --- msa/js-executor/api/jsInvokeMessageProcessor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index bef70936ec..aa8f5d2db0 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -196,8 +196,9 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scriptBody) { var self = this; return new Promise(function (resolve, reject) { - if (self.scriptMap.has(scriptId)) { - resolve(self.scriptMap.get(scriptId)); + const script = self.scriptMap.get(scriptId); + if (script !== undefined) { + resolve(script); } else { self.executor.compileScript(scriptBody).then( (script) => { From ff7fa6237f1f342357995c32488f542c984583d9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 2 Jun 2021 11:05:32 +0300 Subject: [PATCH 13/76] js-executor: zero check and code cleanup --- msa/js-executor/api/jsInvokeMessageProcessor.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index aa8f5d2db0..5257a03168 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -121,9 +121,9 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, r logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; if (this.executedScriptsCounter % statFrequency == 0) { - var nowMs = performance.now(); - var msSinceLastStat = nowMs - this.lastStatTime; - var requestsPerSec = statFrequency / msSinceLastStat * 1000; //msSinceLastStat can't be zero in the real world + const nowMs = performance.now(); + const msSinceLastStat = nowMs - this.lastStatTime; + const requestsPerSec = msSinceLastStat == 0 ? statFrequency : statFrequency / msSinceLastStat * 1000; this.lastStatTime = nowMs; logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s]', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec); } @@ -197,13 +197,13 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scri var self = this; return new Promise(function (resolve, reject) { const script = self.scriptMap.get(scriptId); - if (script !== undefined) { + if (script) { resolve(script); } else { self.executor.compileScript(scriptBody).then( - (script) => { - self.cacheScript(scriptId, script); - resolve(script); + (compiledScript) => { + self.cacheScript(scriptId, compiledScript); + resolve(compiledScript); }, (err) => { reject(err); From dd748245706903847ba72f7c6066e97e026ebe28 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 9 Jun 2021 13:30:28 +0300 Subject: [PATCH 14/76] js-executor: Refactoring Kafka executor for use async/await --- msa/js-executor/queue/kafkaTemplate.js | 68 +++++++++++--------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 7fce429d23..b34727c977 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -36,11 +36,10 @@ let producer; const configEntries = []; let batchMessages = []; -let batchResolvers = []; let sendLoopInstance; function KafkaProducer() { - this.send = (responseTopic, scriptId, rawResponse, headers) => { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { logger.debug('Pending queue response, scriptId: [%s]', scriptId); const message = { topic: responseTopic, @@ -51,60 +50,50 @@ function KafkaProducer() { }] }; - return pushMessageToSendLater(message); + await pushMessageToSendLater(message); } } -function pushMessageToSendLater(message) { - let resolver; - const promise = new Promise((resolve, reject) => { - resolver = resolve; - }); +async function pushMessageToSendLater(message) { batchMessages.push(message); - batchResolvers.push(resolver); if (batchMessages.length >= maxBatchSize) { - sendMessagesAsBatch(); - sendLoopWithLinger(); //reset loop function and reschedule new linger + await sendMessagesAsBatch(true); } - return promise; } function sendLoopWithLinger() { if (sendLoopInstance) { - logger.debug("Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); - clearInterval(sendLoopInstance); + clearTimeout(sendLoopInstance); } else { logger.debug("Starting new send loop with linger [%s]", linger) } - sendLoopInstance = setInterval(sendMessagesAsBatch, linger); + sendLoopInstance = setTimeout(sendMessagesAsBatch, linger); } -function sendMessagesAsBatch() { +async function sendMessagesAsBatch(isImmediately) { + if (sendLoopInstance) { + logger.debug("sendMessagesAsBatch: Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger); + clearTimeout(sendLoopInstance); + } + sendLoopInstance = null; if (batchMessages.length > 0) { - logger.debug('sendMessagesAsBatch, length: [%s]', batchMessages.length); + logger.debug('sendMessagesAsBatch, length: [%s], %s', batchMessages.length, isImmediately ? 'immediately' : ''); const messagesToSend = batchMessages; - const resolvers = batchResolvers; batchMessages = []; - batchResolvers = []; - producer.sendBatch({ - topicMessages: messagesToSend, - acks: acks, - compression: compressionType - }).then( - () => { - logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); - for (let i = 0; i < resolvers.length; i++) { - resolvers[i](); - } - }, - (err) => { - logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); - logger.error(err.stack); - batchMessages = messagesToSend.concat(batchMessages); - batchResolvers = resolvers.concat(batchResolvers); //promises will never be rejected. Will retry forever - } - ); + try { + await producer.sendBatch({ + topicMessages: messagesToSend, + acks: acks, + compression: compressionType + }) + logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length); + } catch(err) { + logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length); + logger.error(err.stack); + batchMessages = messagesToSend.concat(batchMessages); + } } + sendLoopWithLinger(); } (async () => { @@ -285,9 +274,8 @@ async function disconnectProducer() { producer = null; try { logger.info('Stopping loop...'); - //TODO: send handle msg - clearInterval(sendLoopInstance); - sendMessagesAsBatch(); + clearTimeout(sendLoopInstance); + await sendMessagesAsBatch(); await _producer.disconnect(); logger.info('Kafka Producer stopped.'); } catch (e) { From a7a7dca3b3eb76166cf8ed93bf764f2c8c5935b2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 8 Jun 2021 20:03:45 +0300 Subject: [PATCH 15/76] js-executor: upgraded libraries to fix vulnerability warning (@google-cloud/pubsub", amqplib) --- msa/js-executor/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 75f009880b..280979afa6 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -13,8 +13,8 @@ }, "dependencies": { "@azure/service-bus": "^1.1.9", - "@google-cloud/pubsub": "^2.5.0", - "amqplib": "^0.6.0", + "@google-cloud/pubsub": "^2.12.0", + "amqplib": "^0.8.0", "aws-sdk": "^2.741.0", "azure-sb": "^0.11.1", "config": "^3.3.1", From 98bea8eaf579f6641bc8a3dfa43639901aca7c91 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 8 Jun 2021 20:14:10 +0300 Subject: [PATCH 16/76] js-executor: http livenessProbe added --- msa/js-executor/api/httpServer.js | 46 +++++++++++++++++++ .../config/custom-environment-variables.yml | 1 + msa/js-executor/config/default.yml | 1 + msa/js-executor/package.json | 1 + msa/js-executor/server.js | 2 + 5 files changed, 51 insertions(+) create mode 100644 msa/js-executor/api/httpServer.js diff --git a/msa/js-executor/api/httpServer.js b/msa/js-executor/api/httpServer.js new file mode 100644 index 0000000000..996638c7c3 --- /dev/null +++ b/msa/js-executor/api/httpServer.js @@ -0,0 +1,46 @@ +/* + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ + +const config = require('config'), + logger = require('../config/logger')._logger('httpServer'), + express = require('express'); + +const httpPort = Number(config.get('http_port')); + +const app = express(); + +app.get('/livenessProbe', async (req, res) => { + const date = new Date(); + const message = { now: date.toISOString() }; + res.send(message); +}) + +app.listen(httpPort, () => logger.info(`Started http endpoint on port ${httpPort}. Please, use /livenessProbe !`)) \ No newline at end of file diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index c5eb15e8b9..7773fdaf0c 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -16,6 +16,7 @@ queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" +http_port: "HTTP_PORT" # /livenessProbe js: response_poll_interval: "REMOTE_JS_RESPONSE_POLL_INTERVAL_MS" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 5a2c47a675..bcefc1c639 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -16,6 +16,7 @@ queue_type: "kafka" request_topic: "js_eval.requests" +http_port: "8888" # /livenessProbe js: response_poll_interval: "25" diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 280979afa6..ba940ee85a 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -18,6 +18,7 @@ "aws-sdk": "^2.741.0", "azure-sb": "^0.11.1", "config": "^3.3.1", + "express": "^4.17.1", "js-yaml": "^3.14.0", "kafkajs": "^1.15.0", "long": "^4.0.0", diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 0c415cc156..7b2e7e59b8 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -51,3 +51,5 @@ switch (serviceType) { process.exit(-1); } +require('./api/httpServer'); + From 05df5e03efbaff6a1a5b48d27dc6327962ecd4b2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Jun 2021 11:26:33 +0300 Subject: [PATCH 17/76] Revert "js-executor: upgraded libraries to fix vulnerability warning (@google-cloud/pubsub", amqplib)" Untested version with breaking changes declared This reverts commit 80f83a86 --- msa/js-executor/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index ba940ee85a..187144da20 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -13,8 +13,8 @@ }, "dependencies": { "@azure/service-bus": "^1.1.9", - "@google-cloud/pubsub": "^2.12.0", - "amqplib": "^0.8.0", + "@google-cloud/pubsub": "^2.5.0", + "amqplib": "^0.6.0", "aws-sdk": "^2.741.0", "azure-sb": "^0.11.1", "config": "^3.3.1", From 3a31b9c5ea7451dce8e5936e196f9fdbbb75eb4c Mon Sep 17 00:00:00 2001 From: Kien Truong Date: Thu, 3 Jun 2021 11:05:02 +0700 Subject: [PATCH 18/76] Fix wrong configuration key for Compression Type Fix #4678 (cherry picked from commit c176ca94aa0f612c3bb3856ae2acf3a399cc74a4) --- msa/js-executor/queue/kafkaTemplate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index b34727c977..7e19f373a8 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -26,7 +26,7 @@ const acks = Number(config.get('kafka.acks')); const maxBatchSize = Number(config.get('kafka.batch_size')); const linger = Number(config.get('kafka.linger_ms')); const requestTimeout = Number(config.get('kafka.requestTimeout')); -const compressionType = (config.get('kafka.requestTimeout') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; +const compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; let kafkaClient; let kafkaAdmin; From 9a4a94621a88dd7e6e329dc1d7767ff75e311e16 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Jun 2021 14:31:51 +0300 Subject: [PATCH 19/76] js-executor: parameter added for Kafka PARTITIONS_CONSUMED_CONCURRENTLY to decrease max latency while scale down replicas --- msa/js-executor/config/custom-environment-variables.yml | 1 + msa/js-executor/config/default.yml | 1 + msa/js-executor/queue/kafkaTemplate.js | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 7773fdaf0c..34903c4541 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -29,6 +29,7 @@ kafka: acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "${TB_KAFKA_BATCH_SIZE:128}" # for producer linger_ms: "${TB_KAFKA_LINGER_MS:1}" # for producer + partitions_consumed_concurrently: "${PARTITIONS_CONSUMED_CONCURRENTLY:1}" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index bcefc1c639..190cf097ed 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -29,6 +29,7 @@ kafka: acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "128" # for producer linger_ms: "1" # for producer + partitions_consumed_concurrently: "1" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "30000" # The default value in kafkajs is: 30000 compression: "gzip" # gzip or uncompressed topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1" diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 7e19f373a8..46ded135e6 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -27,6 +27,7 @@ const maxBatchSize = Number(config.get('kafka.batch_size')); const linger = Number(config.get('kafka.linger_ms')); const requestTimeout = Number(config.get('kafka.requestTimeout')); const compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; +const partitionsConsumedConcurrently = Number(config.get('kafka.partitions_consumed_concurrently')); let kafkaClient; let kafkaAdmin; @@ -197,7 +198,7 @@ async function sendMessagesAsBatch(isImmediately) { logger.info('Started ThingsBoard JavaScript Executor Microservice.'); await consumer.run({ - //partitionsConsumedConcurrently: 1, // Default: 1 + partitionsConsumedConcurrently: partitionsConsumedConcurrently, eachMessage: async ({topic, partition, message}) => { let headers = message.headers; let key = message.key; From 9ffa50f4a528d3f7b99e167e3bd797213ed26314 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Jun 2021 16:05:50 +0300 Subject: [PATCH 20/76] js-executor: fixed env variables mapping --- msa/js-executor/config/custom-environment-variables.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 34903c4541..4185e65835 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -27,9 +27,9 @@ kafka: servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge - batch_size: "${TB_KAFKA_BATCH_SIZE:128}" # for producer - linger_ms: "${TB_KAFKA_LINGER_MS:1}" # for producer - partitions_consumed_concurrently: "${PARTITIONS_CONSUMED_CONCURRENTLY:1}" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency + batch_size: "TB_KAFKA_BATCH_SIZE" # for producer + linger_ms: "TB_KAFKA_LINGER_MS" # for producer + partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" From 4e15160904952bfbda05d78fa022d6c07a8b2dd8 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 16 Jun 2021 11:17:46 +0300 Subject: [PATCH 21/76] UI: Add dashboard logo sanitizer url; Added dashboard logo cache --- .../dashboard-page.component.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts index cc19e98a99..5f3478bccc 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts @@ -128,6 +128,7 @@ import { DashboardImageDialogComponent, DashboardImageDialogData, DashboardImageDialogResult } from '@home/components/dashboard-page/dashboard-image-dialog.component'; +import { DomSanitizer, SafeUrl } from '@angular/platform-browser'; // @dynamic @Component({ @@ -211,7 +212,8 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC addingLayoutCtx: DashboardPageLayoutContext; - logo = 'assets/logo_title_white.svg'; + private dashboardLogoCache: SafeUrl; + private defaultDashboardLogo = 'assets/logo_title_white.svg'; dashboardCtx: DashboardContext = { instanceId: this.utils.guid(), @@ -312,7 +314,8 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC private ngZone: NgZone, private overlay: Overlay, private viewContainerRef: ViewContainerRef, - private cd: ChangeDetectorRef) { + private cd: ChangeDetectorRef, + private sanitizer: DomSanitizer) { super(store); } @@ -413,6 +416,7 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC private reset() { this.dashboard = null; this.dashboardConfiguration = null; + this.dashboardLogoCache = undefined; this.prevDashboard = null; this.widgetEditMode = false; @@ -570,8 +574,12 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC } } - public get dashboardLogo(): string { - return this.dashboard.configuration.settings.dashboardLogoUrl || this.logo; + public get dashboardLogo(): SafeUrl { + if (!this.dashboardLogoCache) { + const logo = this.dashboard.configuration.settings.dashboardLogoUrl || this.defaultDashboardLogo; + this.dashboardLogoCache = this.sanitizer.bypassSecurityTrustUrl(logo); + } + return this.dashboardLogoCache; } public showRightLayoutSwitch(): boolean { @@ -702,6 +710,7 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC }).afterClosed().subscribe((data) => { if (data) { this.dashboard.configuration.settings = data.settings; + this.dashboardLogoCache = undefined; const newGridSettings = data.gridSettings; if (newGridSettings) { const layout = this.dashboard.configuration.states[layoutKeys.state].layouts[layoutKeys.layout]; @@ -855,11 +864,13 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC if (this.widgetEditMode) { if (revert) { this.dashboard = this.prevDashboard; + this.dashboardLogoCache = undefined; } } else { this.resetHighlight(); if (revert) { this.dashboard = this.prevDashboard; + this.dashboardLogoCache = undefined; this.dashboardConfiguration = this.dashboard.configuration; this.dashboardCtx.dashboardTimewindow = this.dashboardConfiguration.timewindow; this.entityAliasesUpdated(); From 9717ef5ff512d2d9148b65ebb6f7d00558ece4d5 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Jun 2021 18:52:07 +0300 Subject: [PATCH 22/76] Fix DefaultTenantProfileConfiguration model --- ui-ngx/src/app/shared/models/tenant.model.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 4ce7acb5c5..badc48bea4 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -49,8 +49,10 @@ export interface DefaultTenantProfileConfiguration { maxRuleNodeExecutionsPerMessage: number; maxEmails: number; maxSms: number; + maxCreatedAlarms: number; defaultStorageTtlDays: number; + alarmsTtlDays: number; } export type TenantProfileConfigurations = DefaultTenantProfileConfiguration; @@ -81,7 +83,9 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxRuleNodeExecutionsPerMessage: 0, maxEmails: 0, maxSms: 0, - defaultStorageTtlDays: 0 + maxCreatedAlarms: 0, + defaultStorageTtlDays: 0, + alarmsTtlDays: 0 }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; break; From 079ae5fd8f17bb7b6f788372e7734bc5e03a8796 Mon Sep 17 00:00:00 2001 From: Vladyslav Prykhodko Date: Wed, 16 Jun 2021 23:06:52 +0300 Subject: [PATCH 23/76] UI: Refactoring resource: add help link; improvement text and view --- .../resources-library-table-config.resolve.ts | 10 +++---- .../resource/resources-library.component.html | 24 ++++++++++++++--- .../resource/resources-library.component.ts | 27 ++++++++++++++----- ui-ngx/src/app/shared/models/constants.ts | 1 + .../src/app/shared/models/resource.models.ts | 4 +-- .../assets/locale/locale.constant-en_US.json | 7 +++-- .../assets/locale/locale.constant-zh_CN.json | 1 - 7 files changed, 54 insertions(+), 20 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts index 089d4b3d82..96745fa47a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts @@ -69,10 +69,10 @@ export class ResourcesLibraryTableConfigResolver implements Resolve true, - onAction: ($event, entity) => this.exportResource($event, entity) + onAction: ($event, entity) => this.downloadResource($event, entity) } ); @@ -118,7 +118,7 @@ export class ResourcesLibraryTableConfigResolver implements Resolve): boolean { switch (action.action) { - case 'uploadResource': - this.exportResource(action.event, action.entity); + case 'downloadResource': + this.downloadResource(action.event, action.entity); return true; } return false; diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html index 5b98fa9c61..1f33d63776 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html @@ -18,16 +18,26 @@
+
+ +
@@ -47,7 +57,7 @@ {{ 'resource.title-required' | translate }} - +
+ + resource.file-name + + +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts index ce26499102..50d565ccc0 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts @@ -30,6 +30,7 @@ import { ResourceTypeTranslationMap } from '@shared/models/resource.models'; import { pairwise, startWith, takeUntil } from 'rxjs/operators'; +import { ActionNotificationShow } from "@core/notification/notification.actions"; @Component({ selector: 'tb-resources-library', @@ -88,26 +89,29 @@ export class ResourcesLibraryComponent extends EntityComponent impleme } buildForm(entity: Resource): FormGroup { - return this.fb.group( + const form = this.fb.group( { + title: [entity ? entity.title : '', []], resourceType: [{ value: entity?.resourceType ? entity.resourceType : ResourceType.LWM2M_MODEL, - disabled: this.isEdit + disabled: !this.isAdd }, [Validators.required]], - data: [entity ? entity.data : null, [Validators.required]], fileName: [entity ? entity.fileName : null, [Validators.required]], - title: [entity ? entity.title : '', []] } ); + if (this.isAdd) { + form.addControl('data', this.fb.control(null, Validators.required)); + } + return form } updateForm(entity: Resource) { - this.entityForm.patchValue({resourceType: entity.resourceType}); if (this.isEdit) { this.entityForm.get('resourceType').disable({emitEvent: false}); + this.entityForm.get('fileName').disable({emitEvent: false}); } this.entityForm.patchValue({ - data: entity.data, + resourceType: entity.resourceType, fileName: entity.fileName, title: entity.title }); @@ -132,4 +136,15 @@ export class ResourcesLibraryComponent extends EntityComponent impleme convertToBase64File(data: string): string { return window.btoa(data); } + + onResourceIdCopied() { + this.store.dispatch(new ActionNotificationShow( + { + message: this.translate.instant('resource.idCopiedMessage'), + type: 'success', + duration: 750, + verticalPosition: 'bottom', + horizontalPosition: 'right' + })); + } } diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index b72e2b196d..56347f1261 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -120,6 +120,7 @@ export const HelpLinks = { entityViews: helpBaseUrl + '/docs/user-guide/ui/entity-views', entitiesImport: helpBaseUrl + '/docs/user-guide/bulk-provisioning', rulechains: helpBaseUrl + '/docs/user-guide/ui/rule-chains', + resources: helpBaseUrl + '/docs/user-guide/ui/resources', dashboards: helpBaseUrl + '/docs/user-guide/ui/dashboards', otaUpdates: helpBaseUrl + '/docs/user-guide/ui/ota-updates', widgetsBundles: helpBaseUrl + '/docs/user-guide/ui/widget-library#bundles', diff --git a/ui-ngx/src/app/shared/models/resource.models.ts b/ui-ngx/src/app/shared/models/resource.models.ts index 082d09ac24..9c478cca0d 100644 --- a/ui-ngx/src/app/shared/models/resource.models.ts +++ b/ui-ngx/src/app/shared/models/resource.models.ts @@ -61,6 +61,6 @@ export interface Resource extends ResourceInfo { } export interface Resources extends ResourceInfo { - data: string|string[]; - fileName: string|string[]; + data: Array; + fileName: Array; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ec084b8a3a..172cc2033d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2309,20 +2309,23 @@ }, "resource": { "add": "Add Resource", + "copyId": "Copy resource Id", "delete": "Delete resource", "delete-resource-text": "Be careful, after the confirmation the resource will become unrecoverable.", "delete-resource-title": "Are you sure you want to delete the resource '{{resourceTitle}}'?", "delete-resources-action-title": "Delete { count, plural, 1 {1 resource} other {# resources} }", "delete-resources-text": "Be careful, after the confirmation all selected resources will be removed.", "delete-resources-title": "Are you sure you want to delete { count, plural, 1 {1 resource} other {# resources} }?", + "download": "Download resource", "drop-file": "Drop a resource file or click to select a file to upload.", "empty": "Resource is empty", - "export": "Export resource", + "file-name": "File name", + "idCopiedMessage": "Resource Id has been copied to clipboard", "no-resource-matching": "No resource matching '{{widgetsBundle}}' were found.", "no-resource-text": "No resources found", "open-widgets-bundle": "Open widgets bundle", "resource": "Resource", - "resource-library-details": "Resource library details", + "resource-library-details": "Resource details", "resource-type": "Resource type", "resources-library": "Resources library", "search": "Search resources", diff --git a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json index 6a60db96fc..cc23f5fc78 100644 --- a/ui-ngx/src/assets/locale/locale.constant-zh_CN.json +++ b/ui-ngx/src/assets/locale/locale.constant-zh_CN.json @@ -1994,7 +1994,6 @@ "delete-resources-title": "确定要删除 { count, plural, 1 {# 个资源} other {# 个资源} }?", "drop-file": "拖拽资源文件或单击以选择要上传的文件。", "empty": "资源为空", - "export": "导出资源", "no-resource-matching": "找不到与 '{{widgetsBundle}}' 匹配的资源。", "no-resource-text": "找不到资源", "open-widgets-bundle": "打开部件库", From cda6f3678b8cfe7a0046d75ff5daeae3287b8312 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 07:27:42 +0300 Subject: [PATCH 24/76] js-executor: fixed license header --- msa/js-executor/api/httpServer.js | 36 +++++++++---------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/msa/js-executor/api/httpServer.js b/msa/js-executor/api/httpServer.js index 996638c7c3..4948c45f61 100644 --- a/msa/js-executor/api/httpServer.js +++ b/msa/js-executor/api/httpServer.js @@ -1,34 +1,18 @@ /* - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - const config = require('config'), logger = require('../config/logger')._logger('httpServer'), express = require('express'); From c07379812b213f3ca249da31285a00124d6345bd Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 07:28:53 +0300 Subject: [PATCH 25/76] js-executor: updated yarn.lock --- msa/js-executor/yarn.lock | 361 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 347 insertions(+), 14 deletions(-) diff --git a/msa/js-executor/yarn.lock b/msa/js-executor/yarn.lock index 404c059c23..0092372b69 100644 --- a/msa/js-executor/yarn.lock +++ b/msa/js-executor/yarn.lock @@ -418,6 +418,14 @@ abort-controller@^3.0.0: dependencies: event-target-shim "^5.0.0" +accepts@~1.3.7: + version "1.3.7" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" + integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== + dependencies: + mime-types "~2.1.24" + negotiator "0.6.2" + agent-base@6: version "6.0.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.1.tgz#808007e4e5867decb0ab6ab2f928fbdb5a596db4" @@ -487,6 +495,11 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" +array-flatten@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" + integrity sha1-ml9pkFGx5wczKPKgCJaLZOopVdI= + array-union@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" @@ -621,6 +634,22 @@ bluebird@^3.5.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +body-parser@1.19.0: + version "1.19.0" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a" + integrity sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw== + dependencies: + bytes "3.1.0" + content-type "~1.0.4" + debug "2.6.9" + depd "~1.1.2" + http-errors "1.7.2" + iconv-lite "0.4.24" + on-finished "~2.3.0" + qs "6.7.0" + raw-body "2.4.0" + type-is "~1.6.17" + boxen@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" @@ -682,6 +711,11 @@ byline@^5.0.0: resolved "https://registry.yarnpkg.com/byline/-/byline-5.0.0.tgz#741c5216468eadc457b03410118ad77de8c1ddb1" integrity sha1-dBxSFkaOrcRXsDQQEYrXfejB3bE= +bytes@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" + integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== + cacheable-request@^6.0.0: version "6.1.0" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" @@ -838,6 +872,28 @@ configstore@^5.0.1: write-file-atomic "^3.0.0" xdg-basedir "^4.0.0" +content-disposition@0.5.3: + version "0.5.3" + resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" + integrity sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g== + dependencies: + safe-buffer "5.1.2" + +content-type@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" + integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== + +cookie-signature@1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" + integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= + +cookie@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba" + integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg== + core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -867,6 +923,13 @@ dateformat@1.0.2-1.2.3: dependencies: ms "^2.1.1" +debug@2.6.9, debug@^2.2.0, debug@~2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + debug@4, debug@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" @@ -874,13 +937,6 @@ debug@4, debug@^4.1.1: dependencies: ms "^2.1.1" -debug@^2.2.0, debug@~2.6.9: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== - dependencies: - ms "2.0.0" - decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -913,6 +969,16 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= +depd@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" + integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= + +destroy@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" + integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -962,6 +1028,11 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: dependencies: safe-buffer "^5.0.1" +ee-first@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" + integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= + emoji-regex@^7.0.1: version "7.0.3" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" @@ -977,6 +1048,11 @@ enabled@2.0.x: resolved "https://registry.yarnpkg.com/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2" integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== +encodeurl@~1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" + integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= + end-of-stream@^1.0.0, end-of-stream@^1.1.0: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -994,6 +1070,11 @@ escape-goat@^2.0.0: resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" integrity sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q== +escape-html@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" + integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg= + escodegen@^1.14.1: version "1.14.3" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" @@ -1021,6 +1102,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +etag@~1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" + integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= + event-target-shim@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" @@ -1041,6 +1127,42 @@ expand-template@^2.0.3: resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== +express@^4.17.1: + version "4.17.1" + resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" + integrity sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g== + dependencies: + accepts "~1.3.7" + array-flatten "1.1.1" + body-parser "1.19.0" + content-disposition "0.5.3" + content-type "~1.0.4" + cookie "0.4.0" + cookie-signature "1.0.6" + debug "2.6.9" + depd "~1.1.2" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + finalhandler "~1.1.2" + fresh "0.5.2" + merge-descriptors "1.0.1" + methods "~1.1.2" + on-finished "~2.3.0" + parseurl "~1.3.3" + path-to-regexp "0.1.7" + proxy-addr "~2.0.5" + qs "6.7.0" + range-parser "~1.2.1" + safe-buffer "5.1.2" + send "0.17.1" + serve-static "1.14.1" + setprototypeof "1.1.1" + statuses "~1.5.0" + type-is "~1.6.18" + utils-merge "1.0.1" + vary "~1.1.2" + extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -1119,6 +1241,19 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +finalhandler@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" + integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== + dependencies: + debug "2.6.9" + encodeurl "~1.0.2" + escape-html "~1.0.3" + on-finished "~2.3.0" + parseurl "~1.3.3" + statuses "~1.5.0" + unpipe "~1.0.0" + find-up@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" @@ -1155,6 +1290,16 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +forwarded@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" + integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== + +fresh@0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" + integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= + from2@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af" @@ -1384,6 +1529,28 @@ http-cache-semantics@^4.0.0: resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== +http-errors@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f" + integrity sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg== + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + +http-errors@~1.7.2: + version "1.7.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" + integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -1401,6 +1568,13 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" +iconv-lite@0.4.24: + version "0.4.24" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== + dependencies: + safer-buffer ">= 2.1.2 < 3" + ieee754@1.1.13, ieee754@^1.1.4: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" @@ -1431,7 +1605,7 @@ inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= -inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: +inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -1449,6 +1623,11 @@ into-stream@^5.1.1: from2 "^2.3.0" p-is-promise "^3.0.0" +ipaddr.js@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" + integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== + is-arrayish@^0.3.1: version "0.3.2" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" @@ -1764,11 +1943,26 @@ make-dir@^3.0.0: dependencies: semver "^6.0.0" +media-typer@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" + integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= + +merge-descriptors@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" + integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= + merge2@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== +methods@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= + micromatch@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.2.tgz#4fcb0999bf9fbc2fcbdd212f6d629b9a56c39259" @@ -1782,6 +1976,11 @@ mime-db@1.44.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== +mime-db@1.48.0: + version "1.48.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" + integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== + mime-types@^2.1.12, mime-types@~2.1.19: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" @@ -1789,6 +1988,18 @@ mime-types@^2.1.12, mime-types@~2.1.19: dependencies: mime-db "1.44.0" +mime-types@~2.1.24: + version "2.1.31" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" + integrity sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg== + dependencies: + mime-db "1.48.0" + +mime@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + mime@^2.2.0: version "2.4.6" resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" @@ -1833,6 +2044,11 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= +ms@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== + ms@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" @@ -1846,6 +2062,11 @@ multistream@^2.1.1: inherits "^2.0.1" readable-stream "^2.0.5" +negotiator@0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" + integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== + node-fetch@^2.3.0, node-fetch@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" @@ -1899,6 +2120,13 @@ object-hash@^2.0.1: resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea" integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg== +on-finished@~2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" + integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= + dependencies: + ee-first "1.1.1" + once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -1974,6 +2202,11 @@ package-json@^6.3.0: registry-url "^5.0.0" semver "^6.2.0" +parseurl@~1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" + integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== + path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" @@ -1984,6 +2217,11 @@ path-parse@^1.0.6: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== +path-to-regexp@0.1.7: + version "0.1.7" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" + integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w= + path-type@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" @@ -2079,6 +2317,14 @@ protobufjs@^6.8.6, protobufjs@^6.9.0: "@types/node" "^13.7.0" long "^4.0.0" +proxy-addr@~2.0.5: + version "2.0.7" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" + integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== + dependencies: + forwarded "0.2.0" + ipaddr.js "1.9.1" + psl@^1.1.28, psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" @@ -2114,6 +2360,11 @@ pupa@^2.0.1: dependencies: escape-goat "^2.0.0" +qs@6.7.0: + version "6.7.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" + integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== + qs@~6.5.2: version "6.5.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -2129,6 +2380,21 @@ querystringify@^2.1.1: resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== +range-parser@~1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + +raw-body@2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.0.tgz#a1ce6fb9c9bc356ca52e89256ab59059e13d0332" + integrity sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q== + dependencies: + bytes "3.1.0" + http-errors "1.7.2" + iconv-lite "0.4.24" + unpipe "1.0.0" + rc@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -2292,17 +2558,17 @@ run-parallel@^1.1.9: resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679" integrity sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q== +safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safe-buffer@~5.1.0, safe-buffer@~5.1.1, safe-buffer@~5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - -safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -2339,11 +2605,45 @@ semver@^7.1.3: resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== +send@0.17.1: + version "0.17.1" + resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" + integrity sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg== + dependencies: + debug "2.6.9" + depd "~1.1.2" + destroy "~1.0.4" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + fresh "0.5.2" + http-errors "~1.7.2" + mime "1.6.0" + ms "2.1.1" + on-finished "~2.3.0" + range-parser "~1.2.1" + statuses "~1.5.0" + +serve-static@1.14.1: + version "1.14.1" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" + integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.17.1" + set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= +setprototypeof@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" + integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== + signal-exit@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" @@ -2391,6 +2691,11 @@ stack-trace@0.0.x: resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" integrity sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA= +"statuses@>= 1.5.0 < 2", statuses@~1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" + integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= + stream-browserify@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-2.0.2.tgz#87521d38a44aa7ee91ce1cd2a47df0cb49dd660b" @@ -2513,6 +2818,11 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" +toidentifier@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" + integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== + touch@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.0.tgz#fe365f5f75ec9ed4e56825e0bb76d24ab74af83b" @@ -2581,6 +2891,14 @@ type-fest@^0.8.1: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== +type-is@~1.6.17, type-is@~1.6.18: + version "1.6.18" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" + integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== + dependencies: + media-typer "0.3.0" + mime-types "~2.1.24" + typedarray-to-buffer@^3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" @@ -2636,6 +2954,11 @@ universalify@^1.0.0: resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d" integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug== +unpipe@1.0.0, unpipe@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" + integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= + update-notifier@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.1.tgz#895fc8562bbe666179500f9f2cebac4f26323746" @@ -2705,6 +3028,11 @@ util@^0.11.1: dependencies: inherits "2.0.3" +utils-merge@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" + integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= + uuid-parse@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/uuid-parse/-/uuid-parse-1.1.0.tgz#7061c5a1384ae0e1f943c538094597e1b5f3a65b" @@ -2735,6 +3063,11 @@ validator@^9.4.1: resolved "https://registry.yarnpkg.com/validator/-/validator-9.4.1.tgz#abf466d398b561cd243050112c6ff1de6cc12663" integrity sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA== +vary@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" + integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= + verror@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" From d117cec09e25c52ea69807a4d4db051505a3b934 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 07:42:21 +0300 Subject: [PATCH 26/76] js-executor: partitions_consumed_concurrently marked as experimental --- msa/js-executor/config/custom-environment-variables.yml | 2 +- msa/js-executor/config/default.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 4185e65835..fdf261e25c 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -29,7 +29,7 @@ kafka: acks: "TB_KAFKA_ACKS" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "TB_KAFKA_BATCH_SIZE" # for producer linger_ms: "TB_KAFKA_LINGER_MS" # for producer - partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency + partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 190cf097ed..7a6e7c2469 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -29,7 +29,7 @@ kafka: acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "128" # for producer linger_ms: "1" # for producer - partitions_consumed_concurrently: "1" # increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency + partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "30000" # The default value in kafkajs is: 30000 compression: "gzip" # gzip or uncompressed topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1" From a3b31337cae7f0b639285855fd1cc050d05cfda4 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 09:09:12 +0300 Subject: [PATCH 27/76] sync method replaced with async executeGenerateAsync for ScriptEngine api. affected Generator node, ruleChainController (merged with ce) --- .../controller/RuleChainController.java | 4 +- .../service/script/RemoteJsInvokeService.java | 8 ++- .../script/RuleNodeJsScriptEngine.java | 19 ++++--- .../rule/engine/api/ScriptEngine.java | 2 +- .../rule/engine/debug/TbMsgGeneratorNode.java | 50 ++++++++++++------- 5 files changed, 54 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index d5fcc2706f..3b9af2f0f2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -74,6 +74,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -86,6 +87,7 @@ public class RuleChainController extends BaseController { public static final String RULE_NODE_ID = "ruleNodeId"; private static final ObjectMapper objectMapper = new ObjectMapper(); + public static final int TIMEOUT = 20; @Autowired private InstallScripts installScripts; @@ -391,7 +393,7 @@ public class RuleChainController extends BaseController { output = msgToOutput(engine.executeUpdate(inMsg)); break; case "generate": - output = msgToOutput(engine.executeGenerate(inMsg)); + output = msgToOutput(engine.executeGenerateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS)); break; case "filter": boolean result = engine.executeFilter(inMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 334a471973..1bc9203333 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -26,6 +25,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -161,6 +161,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override protected ListenableFuture doInvokeFunction(UUID scriptId, String functionName, Object[] args) { + log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptId, maxRequestsTimeout); String scriptBody = scriptIdToBodysMap.get(scriptId); if (scriptBody == null) { return Futures.immediateFailedFuture(new RuntimeException("No script body found for scriptId: [" + scriptId + "]!")); @@ -180,6 +181,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); if (maxRequestsTimeout > 0) { future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService); @@ -201,6 +205,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }, callbackExecutor); return Futures.transform(future, response -> { + stopWatch.stop(); + log.trace("doInvokeFunction js-response took {}ms for uuid {}", stopWatch.getTotalTimeMillis(), response.getKey()); JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 066ce71a58..c688d6978d 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -102,7 +102,6 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); return TbMsg.transformMsg(msg, newMessageType, msg.getOriginator(), newMetadata, newData); } catch (Throwable th) { - th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); } } @@ -141,13 +140,16 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } @Override - public TbMsg executeGenerate(TbMsg prevMsg) throws ScriptException { - JsonNode result = executeScript(prevMsg); - if (!result.isObject()) { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); - } - return unbindMsg(result, prevMsg); + public ListenableFuture executeGenerateAsync(TbMsg prevMsg) { + log.trace("execute generate async, prevMsg {}", prevMsg); + return Futures.transformAsync(executeScriptAsync(prevMsg), result -> { + if (!result.isObject()) { + log.warn("Wrong result type: {}", result.getNodeType()); + throw new ScriptException("Wrong result type: " + result.getNodeType()); + } + return Futures.immediateFuture(unbindMsg(result, prevMsg)); + }, MoreExecutors.directExecutor()); + } @Override @@ -234,6 +236,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } private ListenableFuture executeScriptAsync(TbMsg msg) { + log.trace("execute script async, msg {}", msg); String[] inArgs = prepareArgs(msg); return Futures.transformAsync(sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]), o -> { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index 49420e8feb..a5dcf535e4 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -29,7 +29,7 @@ public interface ScriptEngine { ListenableFuture> executeUpdateAsync(TbMsg msg); - TbMsg executeGenerate(TbMsg prevMsg) throws ScriptException; + ListenableFuture executeGenerateAsync(TbMsg prevMsg); boolean executeFilter(TbMsg msg) throws ScriptException; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index b5a47b4a0d..8fa4b97063 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -15,9 +15,12 @@ */ package org.thingsboard.rule.engine.debug; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.util.StringUtils; +import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.TbContext; @@ -35,6 +38,7 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -64,10 +68,11 @@ public class TbMsgGeneratorNode implements TbNode { private EntityId originatorId; private UUID nextTickId; private TbMsg prevMsg; - private volatile boolean initialized; + private final AtomicBoolean initialized = new AtomicBoolean(false); @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + log.trace("init generator with config {}", configuration); this.config = TbNodeUtils.convert(configuration, TbMsgGeneratorNodeConfiguration.class); this.delay = TimeUnit.SECONDS.toMillis(config.getPeriodInSeconds()); this.currentMsgCount = 0; @@ -81,35 +86,39 @@ public class TbMsgGeneratorNode implements TbNode { @Override public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { + log.trace("onPartitionChangeMsg, PartitionChangeMsg {}, config {}", msg, config); updateGeneratorState(ctx); } private void updateGeneratorState(TbContext ctx) { + log.trace("updateGeneratorState, config {}", config); if (ctx.isLocalEntity(originatorId)) { - if (!initialized) { - initialized = true; + if (initialized.compareAndSet(false, true)) { this.jsEngine = ctx.createJsScriptEngine(config.getJsScript(), "prevMsg", "prevMetadata", "prevMsgType"); scheduleTickMsg(ctx); } - } else if (initialized) { - initialized = false; + } else if (initialized.compareAndSet(true, false)) { destroy(); } } @Override public void onMsg(TbContext ctx, TbMsg msg) { - if (initialized && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) { + log.trace("onMsg, config {}, msg {}", config, msg); + if (initialized.get() && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) { + TbStopWatch sw = TbStopWatch.startNew(); withCallback(generate(ctx, msg), m -> { - if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { + log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); + if (initialized.get() && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { ctx.enqueueForTellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } }, t -> { - if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { + log.warn("onMsg onFailure callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); + if (initialized.get() && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { ctx.tellFailure(msg, t); scheduleTickMsg(ctx); currentMsgCount++; @@ -119,6 +128,7 @@ public class TbMsgGeneratorNode implements TbNode { } private void scheduleTickMsg(TbContext ctx) { + log.trace("scheduleTickMsg, config {}", config); long curTs = System.currentTimeMillis(); if (lastScheduledTs == 0L) { lastScheduledTs = curTs; @@ -131,22 +141,26 @@ public class TbMsgGeneratorNode implements TbNode { } private ListenableFuture generate(TbContext ctx, TbMsg msg) { - return ctx.getJsExecutor().executeAsync(() -> { - if (prevMsg == null) { - prevMsg = ctx.newMsg(ServiceQueue.MAIN, "", originatorId, msg.getCustomerId(), new TbMsgMetaData(), "{}"); - } - if (initialized) { - ctx.logJsEvalRequest(); - TbMsg generated = jsEngine.executeGenerate(prevMsg); + log.trace("generate, config {}", config); + if (prevMsg == null) { + prevMsg = ctx.newMsg(ServiceQueue.MAIN, "", originatorId, msg.getCustomerId(), new TbMsgMetaData(), "{}"); + } + if (initialized.get()) { + ctx.logJsEvalRequest(); + return Futures.transformAsync(jsEngine.executeGenerateAsync(prevMsg), generated -> { + log.trace("generate process response, generated {}, config {}", generated, config); ctx.logJsEvalResponse(); prevMsg = ctx.newMsg(ServiceQueue.MAIN, generated.getType(), originatorId, msg.getCustomerId(), generated.getMetaData(), generated.getData()); - } - return prevMsg; - }); + return Futures.immediateFuture(prevMsg); + }, MoreExecutors.directExecutor()); + } + return Futures.immediateFuture(prevMsg); + } @Override public void destroy() { + log.trace("destroy, config {}", config); prevMsg = null; if (jsEngine != null) { jsEngine.destroy(); From 6888a1b349de234db1f6f3a8b81326fb34f438b1 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 08:22:38 +0300 Subject: [PATCH 28/76] new utility class tbStopWatch added to improve performance measurement experience --- common/util/pom.xml | 5 ++ .../thingsboard/common/util/TbStopWatch.java | 62 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java diff --git a/common/util/pom.xml b/common/util/pom.xml index 6f66da5790..128bc478d5 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -36,6 +36,11 @@ + + org.springframework + spring-core + ${spring.version} + com.google.guava guava diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java new file mode 100644 index 0000000000..b7dbd0052c --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -0,0 +1,62 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.common.util; + +import org.springframework.util.StopWatch; + +/** + * Utility method that extends Spring Framework StopWatch + * It is a MONOTONIC time stopwatch. + * It is a replacement for any measurements with a wall-clock like System.currentTimeMillis() + * It is not affected by leap second, day-light saving and wall-clock adjustments by manual or network time synchronization + * The main features is a single call for common use cases: + * - create and start: TbStopWatch sw = TbStopWatch.startNew() + * - stop and get: sw.stopAndGetTotalTimeMillis() or sw.stopAndGetLastTaskTimeMillis() + * */ +public class TbStopWatch extends StopWatch { + + public static TbStopWatch startNew(){ + TbStopWatch stopWatch = new TbStopWatch(); + stopWatch.start(); + return stopWatch; + } + + public long stopAndGetTotalTimeMillis(){ + stop(); + return getTotalTimeMillis(); + } + + public long stopAndGetLastTaskTimeMillis(){ + stop(); + return getLastTaskTimeMillis(); + } + +} From f4c98595cb973f1c74f76c647d7bf4c4334851e9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 11:07:59 +0300 Subject: [PATCH 29/76] generator node: comment added on direct executor --- .../org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 8fa4b97063..ab9c68e41f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -152,7 +152,7 @@ public class TbMsgGeneratorNode implements TbNode { ctx.logJsEvalResponse(); prevMsg = ctx.newMsg(ServiceQueue.MAIN, generated.getType(), originatorId, msg.getCustomerId(), generated.getMetaData(), generated.getData()); return Futures.immediateFuture(prevMsg); - }, MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); //usually it runs on js-executor-remote-callback thread pool } return Futures.immediateFuture(prevMsg); From 585f473bdaa9477fd25616982dcc34ba80d50ed9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 15:00:00 +0300 Subject: [PATCH 30/76] exception message added onFailure for GeneratorNode --- .../org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index ab9c68e41f..cb9e25d34b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -117,7 +117,7 @@ public class TbMsgGeneratorNode implements TbNode { } }, t -> { - log.warn("onMsg onFailure callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); + log.warn("onMsg onFailure callback, took {}ms, config {}, msg {}, exception {}", sw.stopAndGetTotalTimeMillis(), config, msg, t); if (initialized.get() && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { ctx.tellFailure(msg, t); scheduleTickMsg(ctx); From a04eac6015bc98e626f446cfa9b6c4be802df626 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 27 Apr 2021 18:26:52 +0300 Subject: [PATCH 31/76] added logs and timing metrics for DefaultTbQueueRequestTemplate, TbKafkaConsumerTemplate (todo revert or refactor) --- .../queue/common/DefaultTbQueueRequestTemplate.java | 8 ++++---- .../server/queue/kafka/TbKafkaConsumerTemplate.java | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index b171f2d8d8..e07be52491 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -92,10 +92,10 @@ public class DefaultTbQueueRequestTemplate responses = responseTemplate.poll(pollInterval); - if (responses.size() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.size()); - } + final int pendingRequestsCount = pendingRequests.size(); + log.trace("Starting template pool topic {}, for pendingRequests {}", responseTemplate.getTopic(), pendingRequestsCount); + List responses = responseTemplate.poll(pollInterval); //poll js responses + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); responses.forEach(response -> { byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); UUID requestId; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index fe4300a421..816a732d41 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.springframework.util.StopWatch; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; @@ -82,7 +83,16 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue @Override protected List> doPoll(long durationInMillis) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + + log.trace("poll topic {} maxDuration {}", getTopic(), durationInMillis); + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + + stopWatch.stop(); + log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + if (records.isEmpty()) { return Collections.emptyList(); } else { From 67c9025a0625270f02626421d5a47d6c6c79ee3c Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 18:01:52 +0300 Subject: [PATCH 32/76] improved logs for timeouts for DefaultTbQueueRequestTemplate --- .../server/queue/common/DefaultTbQueueRequestTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e07be52491..084b43f4b6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -121,7 +121,7 @@ public class DefaultTbQueueRequestTemplate staleRequest = pendingRequests.remove(key); if (staleRequest != null) { - log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); staleRequest.future.setException(new TimeoutException()); } } @@ -129,7 +129,7 @@ public class DefaultTbQueueRequestTemplate Date: Wed, 5 May 2021 20:10:51 +0300 Subject: [PATCH 33/76] refactored for tests DefaultTbQueueRequestTemplate class --- .../common/DefaultTbQueueRequestTemplate.java | 112 ++++++++++-------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 084b43f4b6..2c779d5d5a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -85,59 +85,75 @@ public class DefaultTbQueueRequestTemplate { - long nextCleanupMs = 0L; - while (!stopped) { - try { - final int pendingRequestsCount = pendingRequests.size(); - log.trace("Starting template pool topic {}, for pendingRequests {}", responseTemplate.getTopic(), pendingRequestsCount); - List responses = responseTemplate.poll(pollInterval); //poll js responses - log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); - responses.forEach(response -> { - byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); - UUID requestId; - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header and body", response); - } else { - requestId = bytesToUuid(requestIdHeader); - log.trace("[{}] Response received: {}", requestId, response); - ResponseMetaData expectedResponse = pendingRequests.remove(requestId); - if (expectedResponse == null) { - log.trace("[{}] Invalid or stale request", requestId); - } else { - expectedResponse.future.set(response); + executor.submit(this::fetchAndProcessResponses); + } + + void fetchAndProcessResponses() { + long nextCleanupMs = 0L; + while (!stopped) { + try { + final int pendingRequestsCount = pendingRequests.size(); + log.trace("Starting template pool topic {}, for pendingRequests {}", responseTemplate.getTopic(), pendingRequestsCount); + List responses = doPoll(); //poll js responses + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); + responses.forEach(this::processResponse); + responseTemplate.commit(); + tickTs = System.currentTimeMillis(); + tickSize = pendingRequests.size(); + if (nextCleanupMs < tickTs) { + //cleanup; + pendingRequests.forEach((key, value) -> { + if (value.expTime < tickTs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + setTimeoutException(key, staleRequest); } } }); - responseTemplate.commit(); - tickTs = System.currentTimeMillis(); - tickSize = pendingRequests.size(); - if (nextCleanupMs < tickTs) { - //cleanup; - pendingRequests.forEach((key, value) -> { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); - staleRequest.future.setException(new TimeoutException()); - } - } - }); - nextCleanupMs = tickTs + maxRequestTimeout; - } - } catch (Throwable e) { - log.warn("Failed to obtain responses from queue. Going to sleep " + pollInterval + "ms", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new responses", e2); - } + nextCleanupMs = tickTs + maxRequestTimeout; } + } catch (Throwable e) { + log.warn("Failed to obtain responses from queue. Going to sleep " + pollInterval + "ms", e); + sleep(); } - }); + } + } + + List doPoll() { + return responseTemplate.poll(pollInterval); + } + + void sleep() { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new responses", e2); + } + } + + void setTimeoutException(UUID key, ResponseMetaData staleRequest) { + log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + staleRequest.future.setException(new TimeoutException()); + } + + void processResponse(Response response) { + byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); + UUID requestId; + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header and body", response); + } else { + requestId = bytesToUuid(requestIdHeader); + log.trace("[{}] Response received: {}", requestId, response); + ResponseMetaData expectedResponse = pendingRequests.remove(requestId); + if (expectedResponse == null) { + log.warn("[{}] Invalid or stale request, response: {}", requestId, response); + } else { + expectedResponse.future.set(response); + } + } } @Override @@ -174,7 +190,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); + log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } @@ -184,7 +200,7 @@ public class DefaultTbQueueRequestTemplate Date: Thu, 6 May 2021 11:48:42 +0300 Subject: [PATCH 34/76] test: added init stop test for DefaultTbQueueRequestTemplate --- .../common/DefaultTbQueueRequestTemplate.java | 22 ++-- .../DefaultTbQueueRequestTemplateTest.java | 113 ++++++++++++++++++ 2 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 2c779d5d5a..a51e143e26 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -31,6 +31,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.common.stats.MessagesStats; +import javax.annotation.Nullable; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -47,14 +48,14 @@ public class DefaultTbQueueRequestTemplate requestTemplate; private final TbQueueConsumer responseTemplate; private final ConcurrentMap> pendingRequests; - private final boolean internalExecutor; + final boolean internalExecutor; private final ExecutorService executor; private final long maxRequestTimeout; private final long maxPendingRequests; private final long pollInterval; - private volatile long tickTs = 0L; - private volatile long tickSize = 0L; - private volatile boolean stopped = false; + volatile long tickTs = 0L; + volatile long tickSize = 0L; + volatile boolean stopped = false; private MessagesStats messagesStats; @@ -65,7 +66,7 @@ public class DefaultTbQueueRequestTemplate requestTemplate; + @Mock + TbQueueConsumer responseTemplate; + + long maxRequestTimeout = 20; + long maxPendingRequests = 10000; + long pollInterval = 25; + String topic = "js-responses-tb-node-0"; + + DefaultTbQueueRequestTemplate inst; + + @Before + public void setUp() throws Exception { + willReturn(topic).given(responseTemplate).getTopic(); + inst = new DefaultTbQueueRequestTemplate( + queueAdmin, requestTemplate, responseTemplate, + maxRequestTimeout, maxPendingRequests, pollInterval, executor); + + } + + @Test + public void givenExternalExecutor_whenStop_thenDoNotShutdownExecutor() { + inst = Mockito.spy(inst); + willDoNothing().given(inst).fetchAndProcessResponses(); + Assert.assertFalse(inst.stopped); + + inst.init(); + Assert.assertNotEquals(inst.tickTs, 0); + Assert.assertFalse(inst.internalExecutor); + verify(queueAdmin, times(1)).createTopicIfNotExists(topic); + verify(requestTemplate, times(1)).init(); + verify(responseTemplate, times(1)).subscribe(); + verify(executor, times(1)).submit(any(Runnable.class)); + + inst.stop(); + Assert.assertTrue(inst.stopped); + verify(responseTemplate, times(1)).unsubscribe(); + verify(requestTemplate, times(1)).stop(); + verify(executor, never()).shutdownNow(); + } + + @Test + public void fetchAndProcessResponses() { + inst.init(); + + inst.stop(); + } +} \ No newline at end of file From 872717828cc0e9338f6d1e986abcb67a56e0d06f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 6 May 2021 14:54:27 +0300 Subject: [PATCH 35/76] test: refactored and added mainLoop test DefaultTbQueueRequestTemplate --- .../DefaultTbQueueRequestTemplateTest.java | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 769b66fcd4..413b90c5b9 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -31,27 +31,27 @@ package org.thingsboard.server.queue.common; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.BDDMockito; import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.internal.util.reflection.Whitebox; import org.mockito.runners.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; +import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -61,16 +61,17 @@ public class DefaultTbQueueRequestTemplateTest { @Mock TbQueueAdmin queueAdmin; @Mock - ExecutorService executor; - @Mock TbQueueProducer requestTemplate; @Mock TbQueueConsumer responseTemplate; + @Mock + ExecutorService executorMock; + ExecutorService executor; + String topic = "js-responses-tb-node-0"; long maxRequestTimeout = 20; long maxPendingRequests = 10000; long pollInterval = 25; - String topic = "js-responses-tb-node-0"; DefaultTbQueueRequestTemplate inst; @@ -79,35 +80,70 @@ public class DefaultTbQueueRequestTemplateTest { willReturn(topic).given(responseTemplate).getTopic(); inst = new DefaultTbQueueRequestTemplate( queueAdmin, requestTemplate, responseTemplate, - maxRequestTimeout, maxPendingRequests, pollInterval, executor); + maxRequestTimeout, maxPendingRequests, pollInterval, executorMock); + + } + @After + public void tearDown() throws Exception { + if (executor != null) { + executor.shutdownNow(); + } } @Test - public void givenExternalExecutor_whenStop_thenDoNotShutdownExecutor() { - inst = Mockito.spy(inst); - willDoNothing().given(inst).fetchAndProcessResponses(); - Assert.assertFalse(inst.stopped); + public void givenInstance_whenVerifyInitialParameters_thenOK() { + assertEquals(maxPendingRequests, inst.maxPendingRequests); + assertEquals(maxRequestTimeout, inst.maxRequestTimeout); + assertEquals(pollInterval, inst.pollInterval); + assertEquals(executorMock, inst.executor); + assertFalse(inst.stopped); + assertFalse(inst.internalExecutor); + } + + @Test + public void givenExternalExecutor_whenInitStop_thenOK() { + inst = spy(inst); + willDoNothing().given(inst).mainLoop(); inst.init(); - Assert.assertNotEquals(inst.tickTs, 0); - Assert.assertFalse(inst.internalExecutor); + assertNotEquals(0, inst.tickTs); + assertEquals(0, inst.nextCleanupMs); verify(queueAdmin, times(1)).createTopicIfNotExists(topic); verify(requestTemplate, times(1)).init(); verify(responseTemplate, times(1)).subscribe(); - verify(executor, times(1)).submit(any(Runnable.class)); + verify(executorMock, times(1)).submit(any(Runnable.class)); inst.stop(); - Assert.assertTrue(inst.stopped); + assertTrue(inst.stopped); verify(responseTemplate, times(1)).unsubscribe(); verify(requestTemplate, times(1)).stop(); - verify(executor, never()).shutdownNow(); + verify(executorMock, never()).shutdownNow(); } @Test - public void fetchAndProcessResponses() { - inst.init(); + public void givenMainLoop_whenLoopFewTimes_thenVerifyInvocationCount() throws InterruptedException { + inst = spy(inst); + executor = inst.createExecutor(); + CountDownLatch latch = new CountDownLatch(5); + willDoNothing().given(inst).sleep(); + willAnswer(invocation -> { + if (latch.getCount() == 1) { + inst.stop(); //stop the loop in natural way + } + if (latch.getCount() == 3 || latch.getCount() == 4) { + latch.countDown(); + throw new RuntimeException("test catch block"); + } + latch.countDown(); + return null; + }).given(inst).fetchAndProcessResponses(); - inst.stop(); + executor.submit(inst::mainLoop); + latch.await(10, TimeUnit.SECONDS); + + verify(inst, times(5)).fetchAndProcessResponses(); + verify(inst, times(2)).sleep(); } + } \ No newline at end of file From d5fffa5002528db63302749b354aff8cd488ce5d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 6 May 2021 15:50:07 +0300 Subject: [PATCH 36/76] test: added failed test case with overflow maxPendingRequests for DefaultTbQueueRequestTemplate.send(). another refactoring for easy mocking. --- .../DefaultTbQueueRequestTemplateTest.java | 53 ++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 413b90c5b9..ef9d0835d2 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -39,17 +39,22 @@ import org.mockito.runners.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.TbQueueProducer; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -70,7 +75,7 @@ public class DefaultTbQueueRequestTemplateTest { ExecutorService executor; String topic = "js-responses-tb-node-0"; long maxRequestTimeout = 20; - long maxPendingRequests = 10000; + long maxPendingRequests = 32; long pollInterval = 25; DefaultTbQueueRequestTemplate inst; @@ -78,9 +83,9 @@ public class DefaultTbQueueRequestTemplateTest { @Before public void setUp() throws Exception { willReturn(topic).given(responseTemplate).getTopic(); - inst = new DefaultTbQueueRequestTemplate( + inst = spy(new DefaultTbQueueRequestTemplate( queueAdmin, requestTemplate, responseTemplate, - maxRequestTimeout, maxPendingRequests, pollInterval, executorMock); + maxRequestTimeout, maxPendingRequests, pollInterval, executorMock)); } @@ -103,7 +108,6 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenExternalExecutor_whenInitStop_thenOK() { - inst = spy(inst); willDoNothing().given(inst).mainLoop(); inst.init(); @@ -123,7 +127,6 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenMainLoop_whenLoopFewTimes_thenVerifyInvocationCount() throws InterruptedException { - inst = spy(inst); executor = inst.createExecutor(); CountDownLatch latch = new CountDownLatch(5); willDoNothing().given(inst).sleep(); @@ -146,4 +149,42 @@ public class DefaultTbQueueRequestTemplateTest { verify(inst, times(2)).sleep(); } + @Test + public void givenMessages_whenSend_thenOK() { + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + inst.init(); + int msgCount = 10; + for (int i = 0; i < msgCount; i++) { + inst.send(getRequestMsgMock()); + } + assertEquals(msgCount, inst.pendingRequests.size()); + verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); + } + + @Test + public void givenMessagesOverMaxPendingRequests_whenSend_thenImmediateFailedFutureForTheOfRequests() { + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + inst.init(); + assertEquals(0, inst.tickSize); + int msgOverflowCount = 10; + for (int i = 0; i < inst.maxPendingRequests; i++) { + assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + } + for (int i = 0; i < msgOverflowCount; i++) { + assertTrue("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future + } + assertEquals(inst.maxPendingRequests, inst.pendingRequests.size()); + verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); + } + + @Test + public void givenNothing_whenFetchAndProcessResponsesWithTimeout_thenFail() { + + } + + TbQueueMsg getRequestMsgMock() { + TbQueueMsg requestMsg = mock(TbQueueMsg.class); + willReturn(mock(TbQueueMsgHeaders.class)).given(requestMsg).getHeaders(); + return requestMsg; + } } \ No newline at end of file From 9666986156a0005f2d934646d625b29c02aa091f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 10:04:41 +0300 Subject: [PATCH 37/76] test: added failed test for FetchAndProcessResponses when request removed as staled too early DefaultTbQueueRequestTemplate --- .../common/DefaultTbQueueRequestTemplate.java | 131 ++++++++++++------ .../DefaultTbQueueRequestTemplateTest.java | 60 ++++++-- 2 files changed, 138 insertions(+), 53 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index a51e143e26..bfda553538 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -47,15 +48,16 @@ public class DefaultTbQueueRequestTemplate requestTemplate; private final TbQueueConsumer responseTemplate; - private final ConcurrentMap> pendingRequests; + final ConcurrentMap> pendingRequests; final boolean internalExecutor; - private final ExecutorService executor; - private final long maxRequestTimeout; - private final long maxPendingRequests; - private final long pollInterval; + final ExecutorService executor; + final long maxRequestTimeout; + final long maxPendingRequests; + final long pollInterval; volatile long tickTs = 0L; volatile long tickSize = 0L; volatile boolean stopped = false; + long nextCleanupMs = 0L; private MessagesStats messagesStats; @@ -75,44 +77,26 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses - log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); - responses.forEach(this::processResponse); - responseTemplate.commit(); - tickTs = System.currentTimeMillis(); - tickSize = pendingRequests.size(); - if (nextCleanupMs < tickTs) { - //cleanup; - pendingRequests.forEach((key, value) -> { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - setTimeoutException(key, staleRequest); - } - } - }); - nextCleanupMs = tickTs + maxRequestTimeout; - } + fetchAndProcessResponses(); } catch (Throwable e) { log.warn("Failed to obtain responses from queue. Going to sleep " + pollInterval + "ms", e); sleep(); @@ -120,6 +104,36 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses + //if (responses.size() > 0) { + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); + //} + responses.forEach(this::processResponse); //this can take a long time + responseTemplate.commit(); + tickTs = getCurrentTime(); + tickSize = pendingRequests.size(); + if (nextCleanupMs < tickTs) { + //cleanup; + pendingRequests.forEach((key, value) -> { + if (value.expTime < tickTs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + setTimeoutException(key, staleRequest, tickTs); + } + } + }); + setupNextCleanup(); + } + } + + void setupNextCleanup() { + nextCleanupMs = tickTs + maxRequestTimeout; + log.info("setupNextCleanup {}", nextCleanupMs); + } + List doPoll() { return responseTemplate.poll(pollInterval); } @@ -132,8 +146,13 @@ public class DefaultTbQueueRequestTemplate staleRequest) { - log.info("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + void setTimeoutException(UUID key, ResponseMetaData staleRequest, long tickTs) { + if (tickTs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { + log.info("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + } else { + log.error("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + } + staleRequest.future.setException(new TimeoutException()); } @@ -144,10 +163,10 @@ public class DefaultTbQueueRequestTemplate expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { - log.warn("[{}] Invalid or stale request, response: {}", requestId, response); + log.warn("[{}] Invalid or stale request, response: {}", requestId, String.valueOf(response).replace("\n", " ")); } else { expectedResponse.future.set(response); } @@ -184,11 +203,22 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); + ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future, currentTime, maxRequestTimeout); + log.info("pending {}", responseMetaData); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); + sendToRequestTemplate(request, requestId, future, responseMetaData); + return future; + } + + long getCurrentTime() { + return System.currentTimeMillis(); + } + + void sendToRequestTemplate(Request request, UUID requestId, SettableFuture future, ResponseMetaData responseMetaData) { + log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, String.valueOf(request).replace("\n", " ")); if (messagesStats != null) { messagesStats.incrementTotal(); } @@ -198,7 +228,7 @@ public class DefaultTbQueueRequestTemplate { + @Getter + static class ResponseMetaData { + private final long submitTime; + private final long timeout; private final long expTime; private final SettableFuture future; - ResponseMetaData(long ts, SettableFuture future) { + ResponseMetaData(long ts, SettableFuture future, long submitTime, long timeout) { + this.submitTime = submitTime; + this.timeout = timeout; this.expTime = ts; this.future = future; } + + @Override + public String toString() { + return "ResponseMetaData{" + + "submitTime=" + submitTime + + ", calculatedExpTime=" + (submitTime + timeout) + + ", expTime=" + expTime + + ", deltaMs=" + (expTime - submitTime) + + ", future=" + future + + '}'; + } } } diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index ef9d0835d2..715b4eab21 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -29,23 +29,29 @@ * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. */ package org.thingsboard.server.queue.common; - +import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.TbQueueProducer; +import java.util.Collections; +import java.util.List; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -54,12 +60,17 @@ import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.hamcrest.MatcherAssert.assertThat; + +@Slf4j @RunWith(MockitoJUnitRunner.class) public class DefaultTbQueueRequestTemplateTest { @@ -74,9 +85,9 @@ public class DefaultTbQueueRequestTemplateTest { ExecutorService executor; String topic = "js-responses-tb-node-0"; - long maxRequestTimeout = 20; + long maxRequestTimeout = 10; long maxPendingRequests = 32; - long pollInterval = 25; + long pollInterval = 5; DefaultTbQueueRequestTemplate inst; @@ -171,20 +182,49 @@ public class DefaultTbQueueRequestTemplateTest { assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only } for (int i = 0; i < msgOverflowCount; i++) { - assertTrue("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future + assertFalse("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future } - assertEquals(inst.maxPendingRequests, inst.pendingRequests.size()); + assertThat(inst.pendingRequests.size(), equalTo(inst.maxPendingRequests)); verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); } @Test - public void givenNothing_whenFetchAndProcessResponsesWithTimeout_thenFail() { + public void givenNothing_whenSendAndFetchAndProcessResponsesWithTimeout_thenFail() { + //given + AtomicLong currentTime = new AtomicLong(); + willAnswer(x -> { + log.info("currentTime={}", currentTime.get()); + return currentTime.get(); + }).given(inst).getCurrentTime(); + inst.init(); + inst.setupNextCleanup(); + willReturn(Collections.emptyList()).given(inst).doPoll(); + willDoNothing().given(inst).processResponse(any()); + + //when + for (int i = 0; i <= inst.maxRequestTimeout*2; i++) { + currentTime.incrementAndGet(); + assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + if (i % (inst.maxRequestTimeout * 3 / 2) == 0) { + inst.fetchAndProcessResponses(); + } + } + + //then + ArgumentCaptor argumentCaptorResp = ArgumentCaptor.forClass(DefaultTbQueueRequestTemplate.ResponseMetaData.class); + ArgumentCaptor argumentCaptorUUID = ArgumentCaptor.forClass(UUID.class); + ArgumentCaptor argumentCaptorLong = ArgumentCaptor.forClass(Long.class); + verify(inst, atLeastOnce()).setTimeoutException(argumentCaptorUUID.capture(), argumentCaptorResp.capture(), argumentCaptorLong.capture()); + + List responseMetaDataList = argumentCaptorResp.getAllValues(); + List tickTsList = argumentCaptorLong.getAllValues(); + for (int i = 0; i < responseMetaDataList.size(); i++) { + assertThat("tickTs >= calculatedExpTime", tickTsList.get(i), greaterThanOrEqualTo(responseMetaDataList.get(i).getSubmitTime() + responseMetaDataList.get(i).getTimeout())); + } } TbQueueMsg getRequestMsgMock() { - TbQueueMsg requestMsg = mock(TbQueueMsg.class); - willReturn(mock(TbQueueMsgHeaders.class)).given(requestMsg).getHeaders(); - return requestMsg; + return mock(TbQueueMsg.class, RETURNS_DEEP_STUBS); } } \ No newline at end of file From 28235732c6140dfc9a5cb81d4c2e39b8ac217532 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 12:07:17 +0300 Subject: [PATCH 38/76] test: fixed class DefaultTbQueueRequestTemplate --- .../common/DefaultTbQueueRequestTemplate.java | 90 +++++++++++-------- .../DefaultTbQueueRequestTemplateTest.java | 37 ++++---- 2 files changed, 73 insertions(+), 54 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index bfda553538..d5179385e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -36,10 +36,12 @@ import javax.annotation.Nullable; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; @Slf4j public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate @@ -48,16 +50,15 @@ public class DefaultTbQueueRequestTemplate requestTemplate; private final TbQueueConsumer responseTemplate; - final ConcurrentMap> pendingRequests; + final ConcurrentHashMap> pendingRequests = new ConcurrentHashMap<>(); final boolean internalExecutor; final ExecutorService executor; - final long maxRequestTimeout; + final long maxRequestTimeoutNs; final long maxPendingRequests; final long pollInterval; - volatile long tickTs = 0L; - volatile long tickSize = 0L; volatile boolean stopped = false; - long nextCleanupMs = 0L; + long nextCleanupNs = 0L; + private final Lock cleanerLock = new ReentrantLock(); private MessagesStats messagesStats; @@ -72,8 +73,7 @@ public class DefaultTbQueueRequestTemplate(); - this.maxRequestTimeout = maxRequestTimeout; + this.maxRequestTimeoutNs = TimeUnit.MILLISECONDS.toNanos(maxRequestTimeout); this.maxPendingRequests = maxPendingRequests; this.pollInterval = pollInterval; this.internalExecutor = (executor == null); @@ -88,7 +88,6 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses //if (responses.size() > 0) { @@ -113,25 +112,36 @@ public class DefaultTbQueueRequestTemplate { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - setTimeoutException(key, staleRequest, tickTs); + tryCleanStaleRequests(); + } + + private boolean tryCleanStaleRequests() { + if (!cleanerLock.tryLock()) { + return false; + } + try { + log.trace("tryCleanStaleRequest..."); + final long currentNs = getCurrentClockNs(); + if (nextCleanupNs < currentNs) { + pendingRequests.forEach((key, value) -> { + if (value.expTime < currentNs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + setTimeoutException(key, staleRequest, currentNs); + } } - } - }); - setupNextCleanup(); + }); + setupNextCleanup(); + } + } finally { + cleanerLock.unlock(); } + return true; } void setupNextCleanup() { - nextCleanupMs = tickTs + maxRequestTimeout; - log.info("setupNextCleanup {}", nextCleanupMs); + nextCleanupNs = getCurrentClockNs() + maxRequestTimeoutNs; + log.info("setupNextCleanup {}", nextCleanupNs); } List doPoll() { @@ -146,11 +156,11 @@ public class DefaultTbQueueRequestTemplate staleRequest, long tickTs) { - if (tickTs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { - log.info("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + void setTimeoutException(UUID key, ResponseMetaData staleRequest, long currentNs) { + if (currentNs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { + log.info("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } else { - log.error("Request timeout detected, tickTs [{}], {}, key [{}]", tickTs, staleRequest, key); + log.error("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } staleRequest.future.setException(new TimeoutException()); @@ -197,23 +207,31 @@ public class DefaultTbQueueRequestTemplate send(Request request) { - if (tickSize > maxPendingRequests) { + if (pendingRequests.mappingCount() >= maxPendingRequests) { + log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance", maxPendingRequests); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } UUID requestId = UUID.randomUUID(); request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); - long currentTime = getCurrentTime(); - request.getHeaders().put(REQUEST_TIME, longToBytes(currentTime)); + request.getHeaders().put(REQUEST_TIME, longToBytes(getCurrentTimeMs())); + long currentClockNs = getCurrentClockNs(); SettableFuture future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future, currentTime, maxRequestTimeout); - log.info("pending {}", responseMetaData); - pendingRequests.putIfAbsent(requestId, responseMetaData); + ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + maxRequestTimeoutNs, future, currentClockNs, maxRequestTimeoutNs); + log.info("pending {}", responseMetaData); //TODO trace + if (pendingRequests.putIfAbsent(requestId, responseMetaData) != null) { + log.warn("Pending request already exists [{}]!", maxPendingRequests); + return Futures.immediateFailedFuture(new RuntimeException("Pending request already exists !" + requestId)); + } sendToRequestTemplate(request, requestId, future, responseMetaData); return future; } - long getCurrentTime() { + long getCurrentClockNs() { + return System.nanoTime(); //MONOTONIC clock instead wall clock + } + + long getCurrentTimeMs() { //Wall clock to send Ts to the an external service return System.currentTimeMillis(); } @@ -261,8 +279,8 @@ public class DefaultTbQueueRequestTemplate { log.info("currentTime={}", currentTime.get()); return currentTime.get(); - }).given(inst).getCurrentTime(); + }).given(inst).getCurrentClockNs(); inst.init(); inst.setupNextCleanup(); willReturn(Collections.emptyList()).given(inst).doPoll(); willDoNothing().given(inst).processResponse(any()); //when - for (int i = 0; i <= inst.maxRequestTimeout*2; i++) { - currentTime.incrementAndGet(); + long stepNs = TimeUnit.MILLISECONDS.toNanos(1); + for (long i = 0; i <= inst.maxRequestTimeoutNs * 2; i = i + stepNs) { + currentTime.addAndGet(stepNs); assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only - if (i % (inst.maxRequestTimeout * 3 / 2) == 0) { + if (i % (inst.maxRequestTimeoutNs * 3 / 2) == 0) { inst.fetchAndProcessResponses(); } } From 928b8f0fd9ca39fe7ebe874c7ea195153779b96d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 12:26:44 +0300 Subject: [PATCH 39/76] test: refactored for assertThat for DefaultTbQueueRequestTemplateTest --- .../common/DefaultTbQueueRequestTemplate.java | 5 ++--- .../DefaultTbQueueRequestTemplateTest.java | 17 ++++++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index d5179385e6..c31dc3a899 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -158,11 +158,10 @@ public class DefaultTbQueueRequestTemplate staleRequest, long currentNs) { if (currentNs >= staleRequest.getSubmitTime() + staleRequest.getTimeout()) { - log.info("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); + log.warn("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } else { log.error("Request timeout detected, currentNs [{}], {}, key [{}]", currentNs, staleRequest, key); } - staleRequest.future.setException(new TimeoutException()); } @@ -218,7 +217,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + maxRequestTimeoutNs, future, currentClockNs, maxRequestTimeoutNs); - log.info("pending {}", responseMetaData); //TODO trace + log.trace("pending {}", responseMetaData); if (pendingRequests.putIfAbsent(requestId, responseMetaData) != null) { log.warn("Pending request already exists [{}]!", maxPendingRequests); return Futures.immediateFailedFuture(new RuntimeException("Pending request already exists !" + requestId)); diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 33dfd82989..261ff9c7be 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -87,7 +87,7 @@ public class DefaultTbQueueRequestTemplateTest { ExecutorService executor; String topic = "js-responses-tb-node-0"; long maxRequestTimeout = 10; - long maxPendingRequests = 1000; + long maxPendingRequests = 32; long pollInterval = 5; DefaultTbQueueRequestTemplate inst; @@ -124,14 +124,14 @@ public class DefaultTbQueueRequestTemplateTest { inst.init(); //assertNotEquals(0, inst.tickTs); - assertEquals(0, inst.nextCleanupNs); + assertThat(inst.nextCleanupNs, equalTo(0L)); verify(queueAdmin, times(1)).createTopicIfNotExists(topic); verify(requestTemplate, times(1)).init(); verify(responseTemplate, times(1)).subscribe(); verify(executorMock, times(1)).submit(any(Runnable.class)); inst.stop(); - assertTrue(inst.stopped); + assertThat(inst.stopped, is(true)); verify(responseTemplate, times(1)).unsubscribe(); verify(requestTemplate, times(1)).stop(); verify(executorMock, never()).shutdownNow(); @@ -165,11 +165,11 @@ public class DefaultTbQueueRequestTemplateTest { public void givenMessages_whenSend_thenOK() { willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); inst.init(); - int msgCount = 10; + final int msgCount = 10; for (int i = 0; i < msgCount; i++) { inst.send(getRequestMsgMock()); } - assertEquals(msgCount, inst.pendingRequests.mappingCount()); + assertThat(inst.pendingRequests.mappingCount(), equalTo((long) msgCount)); verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); } @@ -179,10 +179,10 @@ public class DefaultTbQueueRequestTemplateTest { inst.init(); int msgOverflowCount = 10; for (int i = 0; i < inst.maxPendingRequests; i++) { - assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + assertThat(inst.send(getRequestMsgMock()).isDone(), is(false)); //SettableFuture future - pending only } for (int i = 0; i < msgOverflowCount; i++) { - assertTrue("max pending requests overflow", inst.send(getRequestMsgMock()).isDone()); //overflow, immediate failed future + assertThat("max pending requests overflow", inst.send(getRequestMsgMock()).isDone(), is(true)); //overflow, immediate failed future } assertThat(inst.pendingRequests.mappingCount(), equalTo(inst.maxPendingRequests)); verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); @@ -205,7 +205,7 @@ public class DefaultTbQueueRequestTemplateTest { long stepNs = TimeUnit.MILLISECONDS.toNanos(1); for (long i = 0; i <= inst.maxRequestTimeoutNs * 2; i = i + stepNs) { currentTime.addAndGet(stepNs); - assertFalse(inst.send(getRequestMsgMock()).isDone()); //SettableFuture future - pending only + assertThat(inst.send(getRequestMsgMock()).isDone(), is(false)); //SettableFuture future - pending only if (i % (inst.maxRequestTimeoutNs * 3 / 2) == 0) { inst.fetchAndProcessResponses(); } @@ -222,7 +222,6 @@ public class DefaultTbQueueRequestTemplateTest { for (int i = 0; i < responseMetaDataList.size(); i++) { assertThat("tickTs >= calculatedExpTime", tickTsList.get(i), greaterThanOrEqualTo(responseMetaDataList.get(i).getSubmitTime() + responseMetaDataList.get(i).getTimeout())); } - } TbQueueMsg getRequestMsgMock() { From 1e066f21569aa7824177a6080577ac83d29ef2a7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 12:41:50 +0300 Subject: [PATCH 40/76] added timeout parameter to the TbQueueRequestTemplate.send( ) --- .../server/queue/TbQueueRequestTemplate.java | 2 ++ .../common/DefaultTbQueueRequestTemplate.java | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index 5dc89a9c26..192f8e1675 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -24,6 +24,8 @@ public interface TbQueueRequestTemplate send(Request request); + ListenableFuture send(Request request, long timeoutNs); + void stop(); void setMessagesStats(MessagesStats messagesStats); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index c31dc3a899..5b03e1bc2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -206,6 +206,11 @@ public class DefaultTbQueueRequestTemplate send(Request request) { + return send(request, this.maxRequestTimeoutNs); + } + + @Override + public ListenableFuture send(Request request, long requestTimeoutNs) { if (pendingRequests.mappingCount() >= maxPendingRequests) { log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance", maxPendingRequests); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); @@ -216,7 +221,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + maxRequestTimeoutNs, future, currentClockNs, maxRequestTimeoutNs); + ResponseMetaData responseMetaData = new ResponseMetaData<>(currentClockNs + requestTimeoutNs, future, currentClockNs, requestTimeoutNs); log.trace("pending {}", responseMetaData); if (pendingRequests.putIfAbsent(requestId, responseMetaData) != null) { log.warn("Pending request already exists [{}]!", maxPendingRequests); @@ -226,11 +231,18 @@ public class DefaultTbQueueRequestTemplate Date: Fri, 7 May 2021 14:07:20 +0300 Subject: [PATCH 41/76] sleep(poolInterval) replaced with parkNanos(1) for DefaultTbQueueRequestTemplate --- .../server/actors/service/DefaultActorService.java | 10 +--------- .../queue/common/DefaultTbQueueRequestTemplate.java | 10 ++++------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 7a28c205bc..6f359c00b0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -102,15 +102,7 @@ public class DefaultActorService extends TbApplicationEventListener staleRequest, long currentNs) { From 16a55b16e4f7c0f678cd10088b4f8ae94c5d5921 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 15:56:45 +0300 Subject: [PATCH 42/76] js invoke service: logs added when script going to disable due to exception (special log for TimeoutException) --- .../service/script/AbstractJsInvokeService.java | 15 ++++++++++++--- .../script/AbstractNashornJsInvokeService.java | 2 +- .../service/script/RemoteJsInvokeService.java | 7 ++++--- .../common/DefaultTbQueueRequestTemplate.java | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 2f0378ae9d..b089f52e76 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -30,6 +30,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; /** @@ -84,8 +85,10 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.JS_EXEC_COUNT, 1); return doInvokeFunction(scriptId, functionName, args); } else { - return Futures.immediateFailedFuture( - new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!")); + String message = "Script invocation is blocked due to maximum error count " + + getMaxErrors() + ", scriptId " + scriptId + "!"; + log.warn(message); + return Futures.immediateFailedFuture(new RuntimeException(message)); } } else { return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!")); @@ -117,7 +120,13 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); - protected void onScriptExecutionError(UUID scriptId) { + protected void onScriptExecutionError(UUID scriptId, Throwable t) { + if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { + log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test + disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).get(), + scriptId); + //return; //timeout is not a good reason to disable function + } disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 9985ac60a2..f9fd3db977 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -160,7 +160,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer return ((Invocable) engine).invokeFunction(functionName, args); } } catch (Exception e) { - onScriptExecutionError(scriptId); + onScriptExecutionError(scriptId, e); throw new ExecutionException(e); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 334a471973..b8bfeea75b 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -193,7 +193,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override public void onFailure(Throwable t) { - onScriptExecutionError(scriptId); + onScriptExecutionError(scriptId, t); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { queueTimeoutMsgs.incrementAndGet(); } @@ -205,9 +205,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { - onScriptExecutionError(scriptId); + final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails()); + onScriptExecutionError(scriptId, e); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); - throw new RuntimeException(invokeResult.getErrorDetails()); + throw e; } }, callbackExecutor); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index f6a5b33289..7b0e7a14cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -109,7 +109,7 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses //if (responses.size() > 0) { - log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); + log.info("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); //TODO reduce verbose after test //} responses.forEach(this::processResponse); //this can take a long time responseTemplate.commit(); From 24eca345768009b674ad827937247bcdd8cba523 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 5 May 2021 08:22:38 +0300 Subject: [PATCH 43/76] new utility class tbStopWatch added to improve performance measurement experience --- common/util/pom.xml | 5 ++ .../thingsboard/common/util/TbStopWatch.java | 62 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java diff --git a/common/util/pom.xml b/common/util/pom.xml index 6f66da5790..128bc478d5 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -36,6 +36,11 @@ + + org.springframework + spring-core + ${spring.version} + com.google.guava guava diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java new file mode 100644 index 0000000000..b7dbd0052c --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -0,0 +1,62 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.common.util; + +import org.springframework.util.StopWatch; + +/** + * Utility method that extends Spring Framework StopWatch + * It is a MONOTONIC time stopwatch. + * It is a replacement for any measurements with a wall-clock like System.currentTimeMillis() + * It is not affected by leap second, day-light saving and wall-clock adjustments by manual or network time synchronization + * The main features is a single call for common use cases: + * - create and start: TbStopWatch sw = TbStopWatch.startNew() + * - stop and get: sw.stopAndGetTotalTimeMillis() or sw.stopAndGetLastTaskTimeMillis() + * */ +public class TbStopWatch extends StopWatch { + + public static TbStopWatch startNew(){ + TbStopWatch stopWatch = new TbStopWatch(); + stopWatch.start(); + return stopWatch; + } + + public long stopAndGetTotalTimeMillis(){ + stop(); + return getTotalTimeMillis(); + } + + public long stopAndGetLastTaskTimeMillis(){ + stop(); + return getLastTaskTimeMillis(); + } + +} From e2aa4be7415790fdf371023fe22a4c25e5fcc605 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 16:26:20 +0300 Subject: [PATCH 44/76] TbStopWatch: methods for nanos added --- .../java/org/thingsboard/common/util/TbStopWatch.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index b7dbd0052c..4f5d1346a9 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -54,9 +54,19 @@ public class TbStopWatch extends StopWatch { return getTotalTimeMillis(); } + public long stopAndGetTotalTimeNanos(){ + stop(); + return getLastTaskTimeNanos(); + } + public long stopAndGetLastTaskTimeMillis(){ stop(); return getLastTaskTimeMillis(); } + public long stopAndGetLastTaskTimeNanos(){ + stop(); + return getLastTaskTimeNanos(); + } + } From 9daa43a115bd584a409ea1279ca4ef6b4b0ae5cb Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 16:34:35 +0300 Subject: [PATCH 45/76] queue request template: sleep on exception shortened according to the stopwatch. test adjusted --- .../queue/common/DefaultTbQueueRequestTemplate.java | 12 +++++++----- .../common/DefaultTbQueueRequestTemplateTest.java | 10 +++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 7b0e7a14cd..a385ef17a8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; @@ -95,11 +96,13 @@ public class DefaultTbQueueRequestTemplate staleRequest, long currentNs) { diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 261ff9c7be..e60e4071f3 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -54,13 +54,13 @@ import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.Matchers.lessThan; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.longThat; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; @@ -141,7 +141,7 @@ public class DefaultTbQueueRequestTemplateTest { public void givenMainLoop_whenLoopFewTimes_thenVerifyInvocationCount() throws InterruptedException { executor = inst.createExecutor(); CountDownLatch latch = new CountDownLatch(5); - willDoNothing().given(inst).sleep(); + willDoNothing().given(inst).sleep(anyLong()); willAnswer(invocation -> { if (latch.getCount() == 1) { inst.stop(); //stop the loop in natural way @@ -158,7 +158,7 @@ public class DefaultTbQueueRequestTemplateTest { latch.await(10, TimeUnit.SECONDS); verify(inst, times(5)).fetchAndProcessResponses(); - verify(inst, times(2)).sleep(); + verify(inst, times(2)).sleep(longThat(lessThan(TimeUnit.MILLISECONDS.toNanos(inst.pollInterval)))); } @Test From c3aec4d077220aef22f369c6d6ee692205b75a47 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 7 May 2021 15:59:42 +0300 Subject: [PATCH 46/76] revert initDispatcherExecutor --- .../server/actors/service/DefaultActorService.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 6f359c00b0..98f739a24b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -102,7 +102,16 @@ public class DefaultActorService extends TbApplicationEventListener Date: Tue, 11 May 2021 08:01:21 +0300 Subject: [PATCH 47/76] log WARN for onScriptExecutionError to prevent silent ban (actually, system going to idle if the script is a part of a rule chain). restored TRACE level on fetchAndProcessResponses --- .../service/script/AbstractJsInvokeService.java | 17 ++++++++++------- .../common/DefaultTbQueueRequestTemplate.java | 6 ++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index b089f52e76..dc01e0d97e 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -121,13 +121,16 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); protected void onScriptExecutionError(UUID scriptId, Throwable t) { - if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { - log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test - disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).get(), - scriptId); - //return; //timeout is not a good reason to disable function - } - disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet(); + DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()); + log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}", + disableListInfo.get(), scriptId, t.getClass(), t.getCause()); +// if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { +// log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test +// disableListInfo.get(), +// scriptId); +// //return; //timeout is not a good reason to disable function +// } + disableListInfo.incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index a385ef17a8..abeb739160 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -109,11 +109,9 @@ public class DefaultTbQueueRequestTemplate responses = doPoll(); //poll js responses - //if (responses.size() > 0) { - log.info("Completed template poll topic {}, for pendingRequests [{}], received [{}]", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); //TODO reduce verbose after test - //} + log.trace("Completed template poll topic {}, for pendingRequests [{}], received [{}] responses", responseTemplate.getTopic(), pendingRequestsCount, responses.size()); responses.forEach(this::processResponse); //this can take a long time responseTemplate.commit(); tryCleanStaleRequests(); From d3e4102282b581b055ec34c0fa555711a19ae472 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 08:11:15 +0300 Subject: [PATCH 48/76] removed version from pom.xml from common/util for spring-core (version already defined through the root dependency management) --- common/util/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/common/util/pom.xml b/common/util/pom.xml index 128bc478d5..8a0fa1705b 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -39,7 +39,6 @@ org.springframework spring-core - ${spring.version} com.google.guava From da65eaab473d1f89131b71b0710d0c3c53e130cf Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 08:20:16 +0300 Subject: [PATCH 49/76] added spring-core for the root dependency management --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index b7d0eb71f6..5f88903559 100755 --- a/pom.xml +++ b/pom.xml @@ -1009,6 +1009,11 @@ + + org.springframework + spring-core + ${spring.version} + org.springframework.boot spring-boot-starter-web From ddfb57db588dc0e1de67dd662af49a50c2ed0cd2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 09:30:58 +0300 Subject: [PATCH 50/76] log detailed error onScriptExecutionError for debug --- .../server/service/script/AbstractJsInvokeService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index dc01e0d97e..8a6aeb399b 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -123,7 +123,8 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected void onScriptExecutionError(UUID scriptId, Throwable t) { DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()); log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}", - disableListInfo.get(), scriptId, t.getClass(), t.getCause()); + disableListInfo.get(), scriptId, t, t.getCause()); + log.error("onScriptExecutionError", t); // if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { // log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test // disableListInfo.get(), From 1d33c09127165907def60b09bca41a11b60d8d85 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 11 May 2021 16:22:10 +0300 Subject: [PATCH 51/76] JsInvokeService - log js body on script failure to investigate easy --- .../server/service/script/AbstractJsInvokeService.java | 7 +++---- .../service/script/AbstractNashornJsInvokeService.java | 2 +- .../server/service/script/RemoteJsInvokeService.java | 10 +++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 8a6aeb399b..cae4a8e05c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -120,11 +120,10 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract long getMaxBlacklistDuration(); - protected void onScriptExecutionError(UUID scriptId, Throwable t) { + protected void onScriptExecutionError(UUID scriptId, Throwable t, String scriptBody) { DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()); - log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}", - disableListInfo.get(), scriptId, t, t.getCause()); - log.error("onScriptExecutionError", t); + log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}", + disableListInfo.get(), scriptId, t, t.getCause(), scriptBody); // if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { // log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test // disableListInfo.get(), diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index f9fd3db977..15a3cf1c15 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -160,7 +160,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer return ((Invocable) engine).invokeFunction(functionName, args); } } catch (Exception e) { - onScriptExecutionError(scriptId, e); + onScriptExecutionError(scriptId, e, functionName); throw new ExecutionException(e); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index b8bfeea75b..5b3b1f5e23 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -161,7 +160,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override protected ListenableFuture doInvokeFunction(UUID scriptId, String functionName, Object[] args) { - String scriptBody = scriptIdToBodysMap.get(scriptId); + log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptId, maxRequestsTimeout); + final String scriptBody = scriptIdToBodysMap.get(scriptId); if (scriptBody == null) { return Futures.immediateFailedFuture(new RuntimeException("No script body found for scriptId: [" + scriptId + "]!")); } @@ -170,7 +170,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setScriptIdLSB(scriptId.getLeastSignificantBits()) .setFunctionName(functionName) .setTimeout((int) maxRequestsTimeout) - .setScriptBody(scriptIdToBodysMap.get(scriptId)); + .setScriptBody(scriptBody); for (Object arg : args) { jsRequestBuilder.addArgs(arg.toString()); @@ -193,7 +193,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override public void onFailure(Throwable t) { - onScriptExecutionError(scriptId, t); + onScriptExecutionError(scriptId, t, scriptBody); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { queueTimeoutMsgs.incrementAndGet(); } @@ -206,7 +206,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { return invokeResult.getResult(); } else { final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails()); - onScriptExecutionError(scriptId, e); + onScriptExecutionError(scriptId, e, scriptBody); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw e; } From 7e551cc92a94aef3727f5732837846c588a4805b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 13 May 2021 20:23:24 +0300 Subject: [PATCH 52/76] log: reduced severity to trace for DefaultTbQueueRequestTemplate.setupNextCleanup() --- .../server/queue/common/DefaultTbQueueRequestTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index abeb739160..7fa3ac5989 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -143,7 +143,7 @@ public class DefaultTbQueueRequestTemplate doPoll() { From ed44ac8f19e14a652978123147f9869e478c2958 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:27:19 +0300 Subject: [PATCH 53/76] test fixed for DefaultTbQueueRequestTemplateTest --- .../queue/common/DefaultTbQueueRequestTemplateTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index e60e4071f3..c73be715fe 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -37,7 +37,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; @@ -55,12 +55,11 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willDoNothing; import static org.mockito.BDDMockito.willReturn; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.longThat; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; @@ -70,6 +69,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.hamcrest.MockitoHamcrest.longThat; @Slf4j @RunWith(MockitoJUnitRunner.class) From 63c906250c39799a8b0142c529d6b21d71d52477 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:27:52 +0300 Subject: [PATCH 54/76] log cleanup DefaultTbQueueRequestTemplate --- .../server/queue/common/DefaultTbQueueRequestTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 7fa3ac5989..7463c11df5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -245,7 +245,7 @@ public class DefaultTbQueueRequestTemplate future, ResponseMetaData responseMetaData) { - log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, String.valueOf(request).replace("\n", " ")); + log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } @@ -255,7 +255,7 @@ public class DefaultTbQueueRequestTemplate Date: Thu, 17 Jun 2021 13:34:06 +0300 Subject: [PATCH 55/76] code cleanup --- .../server/actors/service/DefaultActorService.java | 1 - .../server/service/script/AbstractJsInvokeService.java | 6 ------ 2 files changed, 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 98f739a24b..7a28c205bc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -111,7 +111,6 @@ public class DefaultActorService extends TbApplicationEventListener new DisableListInfo()); log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}", disableListInfo.get(), scriptId, t, t.getCause(), scriptBody); -// if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { -// log.warn("Script has TimeoutException and will increment counter {} on disabledFunctions for id {}", //TODO remove after test -// disableListInfo.get(), -// scriptId); -// //return; //timeout is not a good reason to disable function -// } disableListInfo.incrementAndGet(); } From 75603e58bec47a8b30292cdaa57b03831547514f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:34:54 +0300 Subject: [PATCH 56/76] TbStopWatch updated licence header for CE --- .../thingsboard/common/util/TbStopWatch.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index 4f5d1346a9..90f58ce7f2 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.common.util; From aaedb9e879ccd1ad09b14d5cc717ec981f6d1927 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:35:20 +0300 Subject: [PATCH 57/76] DefaultTbQueueRequestTemplateTest updated licence header for CE --- .../DefaultTbQueueRequestTemplateTest.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index c73be715fe..e50e204adb 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.server.queue.common; From bec228bc83d1541ed69c92ebb77fd13bb606eb47 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:42:37 +0300 Subject: [PATCH 58/76] test: removed unnecessary stubbing on DefaultTbQueueRequestTemplateTest --- .../queue/common/DefaultTbQueueRequestTemplateTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index e50e204adb..9979e9ac43 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -105,10 +105,7 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenExternalExecutor_whenInitStop_thenOK() { - willDoNothing().given(inst).mainLoop(); - inst.init(); - //assertNotEquals(0, inst.tickTs); assertThat(inst.nextCleanupNs, equalTo(0L)); verify(queueAdmin, times(1)).createTopicIfNotExists(topic); verify(requestTemplate, times(1)).init(); @@ -184,7 +181,6 @@ public class DefaultTbQueueRequestTemplateTest { inst.init(); inst.setupNextCleanup(); willReturn(Collections.emptyList()).given(inst).doPoll(); - willDoNothing().given(inst).processResponse(any()); //when long stepNs = TimeUnit.MILLISECONDS.toNanos(1); From 8915b279a027b16c1d45d2e9728ee5dcf474fa7d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Jun 2021 12:40:38 +0300 Subject: [PATCH 59/76] js-script-engine-api: executeSwitch replaced with asynchronous executeSwitchAsync --- .../controller/RuleChainController.java | 2 +- .../script/RuleNodeJsScriptEngine.java | 12 ++++++--- .../rule/engine/api/ScriptEngine.java | 2 +- .../rule/engine/filter/TbJsSwitchNode.java | 27 ++++++++++++------- .../engine/filter/TbJsSwitchNodeTest.java | 2 +- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 3b9af2f0f2..39692922c1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -400,7 +400,7 @@ public class RuleChainController extends BaseController { output = Boolean.toString(result); break; case "switch": - Set states = engine.executeSwitch(inMsg); + Set states = engine.executeSwitchAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); output = objectMapper.writeValueAsString(states); break; case "json": diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index c688d6978d..e9cf8d621f 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -195,9 +195,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S }, MoreExecutors.directExecutor()); } - @Override - public Set executeSwitch(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); + Set executeSwitchPostProcessFunction(JsonNode result) throws ScriptException { if (result.isTextual()) { return Collections.singleton(result.asText()); } else if (result.isArray()) { @@ -217,6 +215,14 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } } + @Override + public ListenableFuture> executeSwitchAsync(TbMsg msg) { + log.trace("execute switch async, msg {}", msg); + return Futures.transformAsync(executeScriptAsync(msg), + result -> Futures.immediateFuture(executeSwitchPostProcessFunction(result)), + MoreExecutors.directExecutor()); //usually runs in a callbackExecutor + } + private JsonNode executeScript(TbMsg msg) throws ScriptException { try { String[] inArgs = prepareArgs(msg); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index a5dcf535e4..da9bd93f2b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -35,7 +35,7 @@ public interface ScriptEngine { ListenableFuture executeFilterAsync(TbMsg msg); - Set executeSwitch(TbMsg msg) throws ScriptException; + ListenableFuture> executeSwitchAsync(TbMsg msg); JsonNode executeJson(TbMsg msg) throws ScriptException; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java index f677b90158..5af4ccb29d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java @@ -15,7 +15,11 @@ */ package org.thingsboard.rule.engine.filter; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -58,17 +62,20 @@ public class TbJsSwitchNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - ListeningExecutor jsExecutor = ctx.getJsExecutor(); ctx.logJsEvalRequest(); - withCallback(jsExecutor.executeAsync(() -> jsEngine.executeSwitch(msg)), - result -> { - ctx.logJsEvalResponse(); - processSwitch(ctx, msg, result); - }, - t -> { - ctx.logJsEvalFailure(); - ctx.tellFailure(msg, t); - }, ctx.getDbCallbackExecutor()); + Futures.addCallback(jsEngine.executeSwitchAsync(msg), new FutureCallback>() { + @Override + public void onSuccess(@Nullable Set result) { + ctx.logJsEvalResponse(); + processSwitch(ctx, msg, result); + } + + @Override + public void onFailure(Throwable t) { + ctx.logJsEvalFailure(); + ctx.tellFailure(msg, t); + } + }, MoreExecutors.directExecutor()); //usually runs in a callbackExecutor } private void processSwitch(TbContext ctx, TbMsg msg, Set nextRelations) { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index cfacf4cb85..fc2e5aa041 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -73,7 +73,7 @@ public class TbJsSwitchNodeTest { TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); - when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); + when(scriptEngine.executeSwitchAsync(msg)).thenReturn(Futures.immediateFuture(Sets.newHashSet("one", "three"))); node.onMsg(ctx, msg); verify(ctx).getJsExecutor(); From f3757ad127cd0029e347d7b5858a99e4eab62f93 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Jun 2021 18:41:18 +0300 Subject: [PATCH 60/76] js-script-engine-api: refactored executeSwitchAsync --- .../script/RuleNodeJsScriptEngine.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index e9cf8d621f..2e6b48693a 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.script; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -32,6 +31,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -145,7 +145,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S return Futures.transformAsync(executeScriptAsync(prevMsg), result -> { if (!result.isObject()) { log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); + Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); } return Futures.immediateFuture(unbindMsg(result, prevMsg)); }, MoreExecutors.directExecutor()); @@ -195,31 +195,31 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S }, MoreExecutors.directExecutor()); } - Set executeSwitchPostProcessFunction(JsonNode result) throws ScriptException { + ListenableFuture> executeSwitchPostProcessAsyncFunction(JsonNode result) { if (result.isTextual()) { - return Collections.singleton(result.asText()); - } else if (result.isArray()) { - Set nextStates = Sets.newHashSet(); + return Futures.immediateFuture(Collections.singleton(result.asText())); + } + if (result.isArray()) { + Set nextStates = new HashSet<>(); for (JsonNode val : result) { if (!val.isTextual()) { log.warn("Wrong result type: {}", val.getNodeType()); - throw new ScriptException("Wrong result type: " + val.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + val.getNodeType())); } else { nextStates.add(val.asText()); } } - return nextStates; - } else { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); + return Futures.immediateFuture(nextStates); } + log.warn("Wrong result type: {}", result.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); } @Override public ListenableFuture> executeSwitchAsync(TbMsg msg) { log.trace("execute switch async, msg {}", msg); return Futures.transformAsync(executeScriptAsync(msg), - result -> Futures.immediateFuture(executeSwitchPostProcessFunction(result)), + this::executeSwitchPostProcessAsyncFunction, MoreExecutors.directExecutor()); //usually runs in a callbackExecutor } From b397dfb518e101558790127b5e7ad9a7e7aef34f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Jun 2021 20:49:46 +0300 Subject: [PATCH 61/76] js-script-engine-api: js sync calls replaced with completely async calls (CE API only) --- .../controller/RuleChainController.java | 8 +- .../script/RuleNodeJsScriptEngine.java | 140 +++++++----------- .../rule/engine/api/ScriptEngine.java | 9 +- .../rule/engine/api/TbContext.java | 4 + .../rule/engine/action/TbLogNode.java | 30 ++-- .../rule/engine/filter/TbJsSwitchNode.java | 2 - .../engine/filter/TbJsSwitchNodeTest.java | 17 +-- 7 files changed, 82 insertions(+), 128 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 39692922c1..fbfd0d595f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -390,13 +390,13 @@ public class RuleChainController extends BaseController { TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); switch (scriptType) { case "update": - output = msgToOutput(engine.executeUpdate(inMsg)); + output = msgToOutput(engine.executeUpdateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS)); break; case "generate": output = msgToOutput(engine.executeGenerateAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS)); break; case "filter": - boolean result = engine.executeFilter(inMsg); + boolean result = engine.executeFilterAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); output = Boolean.toString(result); break; case "switch": @@ -404,11 +404,11 @@ public class RuleChainController extends BaseController { output = objectMapper.writeValueAsString(states); break; case "json": - JsonNode json = engine.executeJson(inMsg); + JsonNode json = engine.executeJsonAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); output = objectMapper.writeValueAsString(json); break; case "string": - output = engine.executeToString(inMsg); + output = engine.executeToStringAsync(inMsg).get(TIMEOUT, TimeUnit.SECONDS); break; default: throw new IllegalArgumentException("Unsupported script type: " + scriptType); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 2e6b48693a..17e999a533 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -106,96 +106,77 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S } } - @Override - public List executeUpdate(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); - if (result.isObject()) { - return Collections.singletonList(unbindMsg(result, msg)); - } else if (result.isArray()){ - List res = new ArrayList<>(result.size()); - result.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg))); - return res; - } else { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); - } - } - @Override public ListenableFuture> executeUpdateAsync(TbMsg msg) { ListenableFuture result = executeScriptAsync(msg); - return Futures.transformAsync(result, json -> { - if (json.isObject()) { - return Futures.immediateFuture(Collections.singletonList(unbindMsg(json, msg))); - } else if (json.isArray()){ - List res = new ArrayList<>(json.size()); - json.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg))); - return Futures.immediateFuture(res); - } - else{ - log.warn("Wrong result type: {}", json.getNodeType()); - return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); - } - }, MoreExecutors.directExecutor()); + return Futures.transformAsync(result, + json -> executeUpdateTransform(msg, json), + MoreExecutors.directExecutor()); + } + + ListenableFuture> executeUpdateTransform(TbMsg msg, JsonNode json) { + if (json.isObject()) { + return Futures.immediateFuture(Collections.singletonList(unbindMsg(json, msg))); + } else if (json.isArray()) { + List res = new ArrayList<>(json.size()); + json.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg))); + return Futures.immediateFuture(res); + } + log.warn("Wrong result type: {}", json.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); } @Override public ListenableFuture executeGenerateAsync(TbMsg prevMsg) { - log.trace("execute generate async, prevMsg {}", prevMsg); - return Futures.transformAsync(executeScriptAsync(prevMsg), result -> { - if (!result.isObject()) { - log.warn("Wrong result type: {}", result.getNodeType()); - Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); - } - return Futures.immediateFuture(unbindMsg(result, prevMsg)); - }, MoreExecutors.directExecutor()); - + return Futures.transformAsync(executeScriptAsync(prevMsg), + result -> executeGenerateTransform(prevMsg, result), + MoreExecutors.directExecutor()); } - @Override - public JsonNode executeJson(TbMsg msg) throws ScriptException { - return executeScript(msg); + ListenableFuture executeGenerateTransform(TbMsg prevMsg, JsonNode result) { + if (!result.isObject()) { + log.warn("Wrong result type: {}", result.getNodeType()); + Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); + } + return Futures.immediateFuture(unbindMsg(result, prevMsg)); } @Override - public ListenableFuture executeJsonAsync(TbMsg msg) throws ScriptException { + public ListenableFuture executeJsonAsync(TbMsg msg) { return executeScriptAsync(msg); } @Override - public String executeToString(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); - if (!result.isTextual()) { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); - } - return result.asText(); + public ListenableFuture executeToStringAsync(TbMsg msg) { + return Futures.transformAsync(executeScriptAsync(msg), + this::executeToStringTransform, + MoreExecutors.directExecutor()); } - @Override - public boolean executeFilter(TbMsg msg) throws ScriptException { - JsonNode result = executeScript(msg); - if (!result.isBoolean()) { - log.warn("Wrong result type: {}", result.getNodeType()); - throw new ScriptException("Wrong result type: " + result.getNodeType()); + ListenableFuture executeToStringTransform(JsonNode result) { + if (result.isTextual()) { + return Futures.immediateFuture(result.asText()); } - return result.asBoolean(); + log.warn("Wrong result type: {}", result.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + result.getNodeType())); } @Override public ListenableFuture executeFilterAsync(TbMsg msg) { - ListenableFuture result = executeScriptAsync(msg); - return Futures.transformAsync(result, json -> { - if (!json.isBoolean()) { - log.warn("Wrong result type: {}", json.getNodeType()); - return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); - } else { - return Futures.immediateFuture(json.asBoolean()); - } - }, MoreExecutors.directExecutor()); + return Futures.transformAsync(executeScriptAsync(msg), + this::executeFilterTransform, + MoreExecutors.directExecutor()); + } + + ListenableFuture executeFilterTransform(JsonNode json) { + if (json.isBoolean()) { + return Futures.immediateFuture(json.asBoolean()); + } + log.warn("Wrong result type: {}", json.getNodeType()); + return Futures.immediateFailedFuture(new ScriptException("Wrong result type: " + json.getNodeType())); } - ListenableFuture> executeSwitchPostProcessAsyncFunction(JsonNode result) { + ListenableFuture> executeSwitchTransform(JsonNode result) { if (result.isTextual()) { return Futures.immediateFuture(Collections.singleton(result.asText())); } @@ -217,34 +198,19 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S @Override public ListenableFuture> executeSwitchAsync(TbMsg msg) { - log.trace("execute switch async, msg {}", msg); return Futures.transformAsync(executeScriptAsync(msg), - this::executeSwitchPostProcessAsyncFunction, + this::executeSwitchTransform, MoreExecutors.directExecutor()); //usually runs in a callbackExecutor } - private JsonNode executeScript(TbMsg msg) throws ScriptException { - try { - String[] inArgs = prepareArgs(msg); - String eval = sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); - return mapper.readTree(eval); - } catch (ExecutionException e) { - if (e.getCause() instanceof ScriptException) { - throw (ScriptException) e.getCause(); - } else if (e.getCause() instanceof RuntimeException) { - throw new ScriptException(e.getCause().getMessage()); - } else { - throw new ScriptException(e); - } - } catch (Exception e) { - throw new ScriptException(e); - } - } - - private ListenableFuture executeScriptAsync(TbMsg msg) { + ListenableFuture executeScriptAsync(TbMsg msg) { log.trace("execute script async, msg {}", msg); String[] inArgs = prepareArgs(msg); - return Futures.transformAsync(sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]), + return executeScriptAsync(msg.getCustomerId(), inArgs[0], inArgs[1], inArgs[2]); + } + + ListenableFuture executeScriptAsync(CustomerId customerId, Object... args) { + return Futures.transformAsync(sandboxService.invokeFunction(tenantId, customerId, this.scriptId, args), o -> { try { return Futures.immediateFuture(mapper.readTree(o.toString())); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index da9bd93f2b..012e135c60 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -19,14 +19,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.msg.TbMsg; -import javax.script.ScriptException; import java.util.List; import java.util.Set; public interface ScriptEngine { - List executeUpdate(TbMsg msg) throws ScriptException; - ListenableFuture> executeUpdateAsync(TbMsg msg); ListenableFuture executeGenerateAsync(TbMsg prevMsg); @@ -37,11 +34,9 @@ public interface ScriptEngine { ListenableFuture> executeSwitchAsync(TbMsg msg); - JsonNode executeJson(TbMsg msg) throws ScriptException; - - ListenableFuture executeJsonAsync(TbMsg msg) throws ScriptException; + ListenableFuture executeJsonAsync(TbMsg msg); - String executeToString(TbMsg msg) throws ScriptException; + ListenableFuture executeToStringAsync(TbMsg msg); void destroy(); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 43e3a5e329..62ed9b0fa0 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -214,6 +214,10 @@ public interface TbContext { EdgeEventService getEdgeEventService(); + /** + * Js script executors call are completely asynchronous + * */ + @Deprecated ListeningExecutor getJsExecutor(); ListeningExecutor getMailExecutor(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 2b2f0d76a4..c981d5062d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -15,8 +15,11 @@ */ package org.thingsboard.rule.engine.action; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.ListeningExecutor; +import org.checkerframework.checker.nullness.qual.Nullable; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.TbContext; @@ -55,18 +58,21 @@ public class TbLogNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - ListeningExecutor jsExecutor = ctx.getJsExecutor(); ctx.logJsEvalRequest(); - withCallback(jsExecutor.executeAsync(() -> jsEngine.executeToString(msg)), - toString -> { - ctx.logJsEvalResponse(); - log.info(toString); - ctx.tellSuccess(msg); - }, - t -> { - ctx.logJsEvalResponse(); - ctx.tellFailure(msg, t); - }); + Futures.addCallback(jsEngine.executeToStringAsync(msg), new FutureCallback() { + @Override + public void onSuccess(@Nullable String result) { + ctx.logJsEvalResponse(); + log.info(result); + ctx.tellSuccess(msg); + } + + @Override + public void onFailure(Throwable t) { + ctx.logJsEvalResponse(); + ctx.tellFailure(msg, t); + } + }, MoreExecutors.directExecutor()); //usually js responses runs on js callback executor } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java index 5af4ccb29d..bc900a0f2d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbJsSwitchNode.java @@ -33,8 +33,6 @@ import org.thingsboard.server.common.msg.TbMsg; import java.util.Set; -import static org.thingsboard.common.util.DonAsynchron.withCallback; - @Slf4j @RuleNode( type = ComponentType.FILTER, diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index fc2e5aa041..4ac81120b0 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -64,7 +64,7 @@ public class TbJsSwitchNodeTest { private RuleNodeId ruleNodeId = new RuleNodeId(Uuids.timeBased()); @Test - public void multipleRoutesAreAllowed() throws TbNodeException, ScriptException { + public void multipleRoutesAreAllowed() throws TbNodeException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("temp", "10"); @@ -72,11 +72,9 @@ public class TbJsSwitchNodeTest { String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); - mockJsExecutor(); when(scriptEngine.executeSwitchAsync(msg)).thenReturn(Futures.immediateFuture(Sets.newHashSet("one", "three"))); node.onMsg(ctx, msg); - verify(ctx).getJsExecutor(); verify(ctx).tellNext(msg, Sets.newHashSet("one", "three")); } @@ -92,19 +90,6 @@ public class TbJsSwitchNodeTest { node.init(ctx, nodeConfiguration); } - @SuppressWarnings("unchecked") - private void mockJsExecutor() { - when(ctx.getJsExecutor()).thenReturn(executor); - doAnswer((Answer>>) invocationOnMock -> { - try { - Callable task = (Callable) (invocationOnMock.getArguments())[0]; - return Futures.immediateFuture((Set) task.call()); - } catch (Throwable th) { - return Futures.immediateFailedFuture(th); - } - }).when(executor).executeAsync(ArgumentMatchers.any(Callable.class)); - } - private void verifyError(TbMsg msg, String message, Class expectedClass) { ArgumentCaptor captor = ArgumentCaptor.forClass(Throwable.class); verify(ctx).tellFailure(same(msg), captor.capture()); From 966b0923e7e293c0bd5fb6aa3f98d43aa9dc93de Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 14:12:33 +0300 Subject: [PATCH 62/76] js-script-engine-api: fixed CE merge --- .../server/service/script/RuleNodeJsScriptEngine.java | 1 + .../main/java/org/thingsboard/rule/engine/api/ScriptEngine.java | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 17e999a533..4ab81702d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -23,6 +23,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbMsg; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java index 012e135c60..7ff63319cd 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/ScriptEngine.java @@ -28,8 +28,6 @@ public interface ScriptEngine { ListenableFuture executeGenerateAsync(TbMsg prevMsg); - boolean executeFilter(TbMsg msg) throws ScriptException; - ListenableFuture executeFilterAsync(TbMsg msg); ListenableFuture> executeSwitchAsync(TbMsg msg); From 6a63b6e31dd02b9b63178f8bdf77be2cc8740b78 Mon Sep 17 00:00:00 2001 From: Dmitriymush Date: Thu, 17 Jun 2021 14:16:46 +0300 Subject: [PATCH 63/76] fix to entity node level --- .../widget/lib/entities-hierarchy-widget.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts index 5fb11d025b..03d97c50fb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-hierarchy-widget.component.ts @@ -443,7 +443,7 @@ export class EntitiesHierarchyWidgetComponent extends PageComponent implements O const dataPageData = subscription.dataPages[0]; const childNodes: HierarchyNavTreeNode[] = []; datasourcesPageData.data.forEach((childDatasource, index) => { - childNodes.push(this.datasourceToNode(childDatasource as HierarchyNodeDatasource, dataPageData.data[index])); + childNodes.push(this.datasourceToNode(childDatasource as HierarchyNodeDatasource, dataPageData.data[index], nodeCtx)); }); nodeCtx.childrenNodesLoaded = true; childrenNodesLoadCb(this.prepareNodes(childNodes)); From 29910fad8f6634e4fa7cfc1db2cf74d2e723f7d8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 17 Jun 2021 13:34:54 +0300 Subject: [PATCH 64/76] TbStopWatch updated licence header for CE --- .../thingsboard/common/util/TbStopWatch.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index b7dbd0052c..155ffb5360 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.common.util; From 75801045cfcdda85b90973d7e3079b22b747de52 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 17 Jun 2021 15:08:00 +0300 Subject: [PATCH 65/76] UI: Add SNMP device profile configuration --- .../home/components/home-components.module.ts | 5 +- ...ofile-transport-configuration.component.ts | 4 +- ...ile-transport-configuration.component.html | 23 -- ...rofile-communication-config.component.html | 72 ++++++ ...rofile-communication-config.component.scss | 44 ++++ ...-profile-communication-config.component.ts | 218 ++++++++++++++++++ ...snmp-device-profile-mapping.component.html | 81 +++++++ ...snmp-device-profile-mapping.component.scss | 46 ++++ .../snmp-device-profile-mapping.component.ts | 165 +++++++++++++ ...ile-transport-configuration.component.html | 46 ++++ ...ofile-transport-configuration.component.ts | 57 +++-- .../snmp-device-profile-transport.module.ts | 38 +++ ui-ngx/src/app/shared/models/constants.ts | 16 ++ ui-ngx/src/app/shared/models/device.models.ts | 37 ++- .../assets/locale/locale.constant-en_US.json | 25 ++ 15 files changed, 834 insertions(+), 43 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html rename ui-ngx/src/app/modules/home/components/profile/device/{ => snpm}/snmp-device-profile-transport-configuration.component.ts (66%) create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 3958830088..d0ad42b62e 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -99,7 +99,6 @@ import { DeviceProfileDialogComponent } from '@home/components/profile/device-pr import { DeviceProfileAutocompleteComponent } from '@home/components/profile/device-profile-autocomplete.component'; import { MqttDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/mqtt-device-profile-transport-configuration.component'; import { CoapDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/coap-device-profile-transport-configuration.component'; -import { SnmpDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/snmp-device-profile-transport-configuration.component'; import { DeviceProfileAlarmsComponent } from '@home/components/profile/alarm/device-profile-alarms.component'; import { DeviceProfileAlarmComponent } from '@home/components/profile/alarm/device-profile-alarm.component'; import { CreateAlarmRulesComponent } from '@home/components/profile/alarm/create-alarm-rules.component'; @@ -143,6 +142,7 @@ import { SecurityConfigLwm2mComponent } from '@home/components/device/security-c import { SecurityConfigLwm2mServerComponent } from '@home/components/device/security-config-lwm2m-server.component'; import { DashboardImageDialogComponent } from '@home/components/dashboard-page/dashboard-image-dialog.component'; import { WidgetContainerComponent } from '@home/components/widget/widget-container.component'; +import { SnmpDeviceProfileTransportModule } from '@home/components/profile/device/snpm/snmp-device-profile-transport.module'; @NgModule({ declarations: @@ -228,7 +228,6 @@ import { WidgetContainerComponent } from '@home/components/widget/widget-contain DefaultDeviceProfileTransportConfigurationComponent, MqttDeviceProfileTransportConfigurationComponent, CoapDeviceProfileTransportConfigurationComponent, - SnmpDeviceProfileTransportConfigurationComponent, DeviceProfileTransportConfigurationComponent, CreateAlarmRulesComponent, AlarmRuleComponent, @@ -272,6 +271,7 @@ import { WidgetContainerComponent } from '@home/components/widget/widget-contain SharedModule, SharedHomeComponentsModule, Lwm2mProfileComponentsModule, + SnmpDeviceProfileTransportModule, StatesControllerModule ], exports: [ @@ -339,7 +339,6 @@ import { WidgetContainerComponent } from '@home/components/widget/widget-contain DefaultDeviceProfileTransportConfigurationComponent, MqttDeviceProfileTransportConfigurationComponent, CoapDeviceProfileTransportConfigurationComponent, - SnmpDeviceProfileTransportConfigurationComponent, DeviceProfileTransportConfigurationComponent, CreateAlarmRulesComponent, AlarmRuleComponent, diff --git a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts index 305a0a84b6..50b3bc801c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.ts @@ -89,7 +89,9 @@ export class DeviceProfileTransportConfigurationComponent implements ControlValu if (configuration) { delete configuration.type; } - this.deviceProfileTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + setTimeout(() => { + this.deviceProfileTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + }); } private updateModel() { diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html deleted file mode 100644 index f57b1ac716..0000000000 --- a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html +++ /dev/null @@ -1,23 +0,0 @@ - -
- - -
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html new file mode 100644 index 0000000000..7d2f8a154c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.html @@ -0,0 +1,72 @@ + +
+
+
+ + device-profile.snmp.scope + + + {{ snmpSpecTypeTranslationMap.get(snmpSpecType) }} + + + + {{ 'device-profile.snmp.scope-required' | translate }} + + + +
+ + device-profile.snmp.querying-frequency + + + {{ 'device-profile.snmp.querying-frequency-required' | translate }} + + + {{ 'device-profile.snmp.querying-frequency-invalid-format' | translate }} + + + + +
+
+ +
+
+ device-profile.snmp.please-add-communication-config +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss new file mode 100644 index 0000000000..3b6661b862 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.scss @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .communication-config { + border: 2px groove rgba(0, 0, 0, 0.25); + border-radius: 4px; + padding: 8px; + min-width: 0; + } + + .scope-row { + padding-bottom: 8px; + } + + .required-text { + margin: 16px 0 + } +} + +:host ::ng-deep { + .mat-form-field.spec { + .mat-form-field-infix { + width: 160px; + } + } + .button-icon{ + font-size: 20px; + width: 20px; + height: 20px; + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts new file mode 100644 index 0000000000..d683d46880 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts @@ -0,0 +1,218 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { SnmpCommunicationConfig, SnmpSpecType, SnmpSpecTypeTranslationMap } from '@shared/models/device.models'; +import { Subject, Subscription } from 'rxjs'; +import { isUndefinedOrNull } from '@core/utils'; +import { takeUntil } from 'rxjs/operators'; + +@Component({ + selector: 'tb-snmp-device-profile-communication-config', + templateUrl: './snmp-device-profile-communication-config.component.html', + styleUrls: ['./snmp-device-profile-communication-config.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceProfileCommunicationConfigComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceProfileCommunicationConfigComponent), + multi: true + }] +}) +export class SnmpDeviceProfileCommunicationConfigComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { + + snmpSpecTypes = Object.values(SnmpSpecType); + snmpSpecTypeTranslationMap = SnmpSpecTypeTranslationMap; + + deviceProfileCommunicationConfig: FormGroup; + + @Input() + disabled: boolean; + + private usedSpecType: SnmpSpecType[] = []; + private valueChange$: Subscription = null; + private destroy$ = new Subject(); + private propagateChange = (v: any) => { }; + + constructor(private fb: FormBuilder) { } + + ngOnInit(): void { + this.deviceProfileCommunicationConfig = this.fb.group({ + communicationConfig: this.fb.array([]) + }); + } + + ngOnDestroy() { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + this.destroy$.next(); + this.destroy$.complete(); + } + + communicationConfigFormArray(): FormArray { + return this.deviceProfileCommunicationConfig.get('communicationConfig') as FormArray; + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean) { + this.disabled = isDisabled; + if (this.disabled) { + this.deviceProfileCommunicationConfig.disable({emitEvent: false}); + } else { + this.deviceProfileCommunicationConfig.enable({emitEvent: false}); + } + } + + writeValue(communicationConfig: SnmpCommunicationConfig[]) { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + const communicationConfigControl: Array = []; + if (communicationConfig) { + communicationConfig.forEach((config) => { + communicationConfigControl.push(this.createdFormGroup(config)); + }); + } + this.deviceProfileCommunicationConfig.setControl('communicationConfig', this.fb.array(communicationConfigControl)); + if (!communicationConfig || !communicationConfig.length) { + this.addCommunicationConfig(); + } + if (this.disabled) { + this.deviceProfileCommunicationConfig.disable({emitEvent: false}); + } else { + this.deviceProfileCommunicationConfig.enable({emitEvent: false}); + } + this.valueChange$ = this.deviceProfileCommunicationConfig.valueChanges.subscribe(() => { + this.updateModel(); + }); + this.updateUsedSpecType(); + if (!this.disabled && !this.deviceProfileCommunicationConfig.valid) { + this.updateModel(); + } + } + + public validate() { + return this.deviceProfileCommunicationConfig.valid && this.deviceProfileCommunicationConfig.value.communicationConfig.length ? null : { + communicationConfig: false + }; + } + + public removeCommunicationConfig(index: number) { + this.communicationConfigFormArray().removeAt(index); + } + + + get isAddEnabled(): boolean { + return this.communicationConfigFormArray().length !== Object.keys(SnmpSpecType).length; + } + + public addCommunicationConfig() { + this.communicationConfigFormArray().push(this.createdFormGroup()); + this.deviceProfileCommunicationConfig.updateValueAndValidity(); + if (!this.deviceProfileCommunicationConfig.valid) { + this.updateModel(); + } + } + + private getFirstUnusedSeverity(): SnmpSpecType { + for (const type of Object.values(SnmpSpecType)) { + if (this.usedSpecType.indexOf(type) === -1) { + return type; + } + } + return null; + } + + public isDisabledSeverity(type: SnmpSpecType, index: number): boolean { + const usedIndex = this.usedSpecType.indexOf(type); + return usedIndex > -1 && usedIndex !== index; + } + + public isShowFrequency(type: SnmpSpecType): boolean { + return type === SnmpSpecType.TELEMETRY_QUERYING || type === SnmpSpecType.CLIENT_ATTRIBUTES_QUERYING; + } + + private updateUsedSpecType() { + this.usedSpecType = []; + const value: SnmpCommunicationConfig[] = this.deviceProfileCommunicationConfig.get('communicationConfig').value; + value.forEach((rule, index) => { + this.usedSpecType[index] = rule.spec; + }); + } + + private createdFormGroup(value?: SnmpCommunicationConfig): FormGroup { + if (isUndefinedOrNull(value)) { + value = { + spec: this.getFirstUnusedSeverity(), + queryingFrequencyMs: 0, + mappings: null + }; + } + const form = this.fb.group({ + spec: [value.spec, Validators.required], + mappings: [value.mappings] + }); + if (this.isShowFrequency(value.spec)) { + form.addControl('queryingFrequencyMs', + this.fb.control(value.queryingFrequencyMs, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); + } + form.get('spec').valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(spec => { + if (this.isShowFrequency(spec)) { + form.addControl('queryingFrequencyMs', + this.fb.control(0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); + } else { + form.removeControl('queryingFrequencyMs'); + } + }); + return form; + } + + private updateModel() { + const value: SnmpCommunicationConfig[] = this.deviceProfileCommunicationConfig.get('communicationConfig').value; + value.forEach(config => { + if (!this.isShowFrequency(config.spec)) { + delete config.queryingFrequencyMs; + } + }); + this.updateUsedSpecType(); + this.propagateChange(value); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html new file mode 100644 index 0000000000..a2a0a97d15 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.html @@ -0,0 +1,81 @@ + +
+
+
+ + + + +
+
+ +
+
+ + + + + {{ dataTypesTranslationMap.get(dataType) | translate }} + + + + {{ 'device-profile.snmp.data-type-required' | translate }} + + + + + + + {{ 'device-profile.snmp.data-key-required' | translate }} + + + + + + + {{ 'device-profile.snmp.oid-required' | translate }} + + + {{ 'device-profile.snmp.oid-pattern' | translate }} + + + +
+
+
+ device-profile.snmp.please-add-mapping-config +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss new file mode 100644 index 0000000000..94cf2290ac --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .mapping-config { + min-width: 518px; + } + .mapping-list { + padding-bottom: 8px; + height: 46px; + } + + .required-text { + margin: 14px 0; + } +} + +:host ::ng-deep { + .mapping-list { + mat-form-field { + .mat-form-field-wrapper { + padding-bottom: 0; + .mat-form-field-infix { + border-top-width: 0.2em; + width: auto; + min-width: auto; + } + .mat-form-field-underline { + bottom: 0; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts new file mode 100644 index 0000000000..53dbec422c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.ts @@ -0,0 +1,165 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { SnmpMapping } from '@shared/models/device.models'; +import { Subscription } from 'rxjs'; +import { DataType, DataTypeTranslationMap } from '@shared/models/constants'; +import { isUndefinedOrNull } from '@core/utils'; + +@Component({ + selector: 'tb-snmp-device-profile-mapping', + templateUrl: './snmp-device-profile-mapping.component.html', + styleUrls: ['./snmp-device-profile-mapping.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceProfileMappingComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceProfileMappingComponent), + multi: true + }] +}) +export class SnmpDeviceProfileMappingComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { + + mappingsConfigForm: FormGroup; + + dataTypes = Object.values(DataType); + dataTypesTranslationMap = DataTypeTranslationMap; + + @Input() + disabled: boolean; + + private readonly oidPattern: RegExp = /^\.?([0-2])((\.0)|(\.[1-9][0-9]*))*$/; + + private valueChange$: Subscription = null; + private propagateChange = (v: any) => { }; + + constructor(private fb: FormBuilder) { } + + ngOnInit() { + this.mappingsConfigForm = this.fb.group({ + mappings: this.fb.array([]) + }); + } + + ngOnDestroy() { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(fn: any) { + } + + setDisabledState(isDisabled: boolean) { + this.disabled = isDisabled; + if (this.disabled) { + this.mappingsConfigForm.disable({emitEvent: false}); + } else { + this.mappingsConfigForm.enable({emitEvent: false}); + } + } + + validate(): ValidationErrors | null { + return this.mappingsConfigForm.valid && this.mappingsConfigForm.value.mappings.length ? null : { + mapping: false + }; + } + + writeValue(mappings: SnmpMapping[]) { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + } + const mappingsControl: Array = []; + if (mappings) { + mappings.forEach((config) => { + mappingsControl.push(this.createdFormGroup(config)); + }); + } + this.mappingsConfigForm.setControl('mappings', this.fb.array(mappingsControl)); + if (!mappings || !mappings.length) { + this.addMappingConfig(); + } + if (this.disabled) { + this.mappingsConfigForm.disable({emitEvent: false}); + } else { + this.mappingsConfigForm.enable({emitEvent: false}); + } + this.valueChange$ = this.mappingsConfigForm.valueChanges.subscribe(() => { + this.updateModel(); + }); + if (!this.disabled && !this.mappingsConfigForm.valid) { + this.updateModel(); + } + } + + mappingsConfigFormArray(): FormArray { + return this.mappingsConfigForm.get('mappings') as FormArray; + } + + public addMappingConfig() { + this.mappingsConfigFormArray().push(this.createdFormGroup()); + this.mappingsConfigForm.updateValueAndValidity(); + if (!this.mappingsConfigForm.valid) { + this.updateModel(); + } + } + + public removeMappingConfig(index: number) { + this.mappingsConfigFormArray().removeAt(index); + } + + private createdFormGroup(value?: SnmpMapping): FormGroup { + if (isUndefinedOrNull(value)) { + value = { + dataType: DataType.STRING, + key: '', + oid: '' + }; + } + return this.fb.group({ + dataType: [value.dataType, Validators.required], + key: [value.key, Validators.required], + oid: [value.oid, [Validators.required, Validators.pattern(this.oidPattern)]] + }); + } + + private updateModel() { + const value: SnmpMapping[] = this.mappingsConfigForm.get('mappings').value; + this.propagateChange(value); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html new file mode 100644 index 0000000000..7dad337d72 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.html @@ -0,0 +1,46 @@ + +
+
+ + device-profile.snmp.timeout-ms + + + {{ 'device-profile.snmp.timeout-ms-required' | translate }} + + + {{ 'device-profile.snmp.timeout-ms-invalid-format' | translate }} + + + + device-profile.snmp.retries + + + {{ 'device-profile.snmp.retries-required' | translate }} + + + {{ 'device-profile.snmp.retries-invalid-format' | translate }} + + +
+
device-profile.snmp.communication-configs
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts similarity index 66% rename from ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts rename to ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts index e6749a9219..a384acbae3 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts @@ -15,9 +15,16 @@ /// import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; -import { Store } from '@ngrx/store'; -import { AppState } from '@app/core/core.state'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { DeviceProfileTransportConfiguration, @@ -40,19 +47,24 @@ export interface OidMappingConfiguration { selector: 'tb-snmp-device-profile-transport-configuration', templateUrl: './snmp-device-profile-transport-configuration.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => SnmpDeviceProfileTransportConfigurationComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceProfileTransportConfigurationComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceProfileTransportConfigurationComponent), + multi: true + }] }) -export class SnmpDeviceProfileTransportConfigurationComponent implements ControlValueAccessor, OnInit, OnDestroy { +export class SnmpDeviceProfileTransportConfigurationComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { snmpDeviceProfileTransportConfigurationFormGroup: FormGroup; private destroy$ = new Subject(); private requiredValue: boolean; - private configuration = []; get required(): boolean { return this.requiredValue; @@ -69,12 +81,14 @@ export class SnmpDeviceProfileTransportConfigurationComponent implements Control private propagateChange = (v: any) => { } - constructor(private store: Store, private fb: FormBuilder) { + constructor(private fb: FormBuilder) { } ngOnInit(): void { this.snmpDeviceProfileTransportConfigurationFormGroup = this.fb.group({ - configuration: [null, Validators.required] + timeoutMs: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + retries: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + communicationConfigs: [null, Validators.required], }); this.snmpDeviceProfileTransportConfigurationFormGroup.valueChanges.pipe( takeUntil(this.destroy$) @@ -95,18 +109,33 @@ export class SnmpDeviceProfileTransportConfigurationComponent implements Control registerOnTouched(fn: any): void { } + setDisabledState(isDisabled: boolean) { + this.disabled = isDisabled; + if (this.disabled) { + this.snmpDeviceProfileTransportConfigurationFormGroup.disable({emitEvent: false}); + } else { + this.snmpDeviceProfileTransportConfigurationFormGroup.enable({emitEvent: false}); + } + } + writeValue(value: SnmpDeviceProfileTransportConfiguration | null): void { if (isDefinedAndNotNull(value)) { - this.snmpDeviceProfileTransportConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + this.snmpDeviceProfileTransportConfigurationFormGroup.patchValue(value, {emitEvent: !value.communicationConfigs}); } } private updateModel() { let configuration: DeviceProfileTransportConfiguration = null; if (this.snmpDeviceProfileTransportConfigurationFormGroup.valid) { - configuration = this.snmpDeviceProfileTransportConfigurationFormGroup.getRawValue().configuration; + configuration = this.snmpDeviceProfileTransportConfigurationFormGroup.getRawValue(); configuration.type = DeviceTransportType.SNMP; } this.propagateChange(configuration); } + + validate(): ValidationErrors | null { + return this.snmpDeviceProfileTransportConfigurationFormGroup.valid ? null : { + snmpDeviceProfileTransportConfiguration: false + }; + } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts new file mode 100644 index 0000000000..c0797d9b0e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport.module.ts @@ -0,0 +1,38 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { SharedModule } from '@shared/shared.module'; +import { CommonModule } from '@angular/common'; +import { SnmpDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component'; +import { SnmpDeviceProfileCommunicationConfigComponent } from './snmp-device-profile-communication-config.component'; +import { SnmpDeviceProfileMappingComponent } from './snmp-device-profile-mapping.component'; + +@NgModule({ + declarations: [ + SnmpDeviceProfileTransportConfigurationComponent, + SnmpDeviceProfileCommunicationConfigComponent, + SnmpDeviceProfileMappingComponent + ], + imports: [ + CommonModule, + SharedModule + ], + exports: [ + SnmpDeviceProfileTransportConfigurationComponent + ] +}) +export class SnmpDeviceProfileTransportModule { } diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index b72e2b196d..cd81753c76 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -147,6 +147,22 @@ export enum ValueType { JSON = 'JSON' } +export enum DataType { + STRING = 'STRING', + LONG = 'LONG', + BOOLEAN = 'BOOLEAN', + DOUBLE = 'DOUBLE', + JSON = 'JSON' +} + +export const DataTypeTranslationMap = new Map([ + [DataType.STRING, 'value.string'], + [DataType.LONG, 'value.integer'], + [DataType.BOOLEAN, 'value.boolean'], + [DataType.DOUBLE, 'value.double'], + [DataType.JSON, 'value.json'] +]); + export const valueTypesMap = new Map( [ [ diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 7af8e56472..568faeb452 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -29,6 +29,7 @@ import * as _moment from 'moment'; import { AbstractControl, ValidationErrors } from '@angular/forms'; import { OtaPackageId } from '@shared/models/id/ota-package-id'; import { DashboardId } from '@shared/models/id/dashboard-id'; +import { DataType } from '@shared/models/constants'; export enum DeviceProfileType { DEFAULT = 'DEFAULT', @@ -257,7 +258,35 @@ export interface Lwm2mDeviceProfileTransportConfiguration { } export interface SnmpDeviceProfileTransportConfiguration { - [key: string]: any; + timeoutMs?: number; + retries?: number; + communicationConfigs?: SnmpCommunicationConfig[]; +} + +export enum SnmpSpecType { + TELEMETRY_QUERYING = 'TELEMETRY_QUERYING', + CLIENT_ATTRIBUTES_QUERYING = 'CLIENT_ATTRIBUTES_QUERYING', + SHARED_ATTRIBUTES_SETTING = 'SHARED_ATTRIBUTES_SETTING', + TO_DEVICE_RPC_REQUEST = 'TO_DEVICE_RPC_REQUEST' +} + +export const SnmpSpecTypeTranslationMap = new Map([ + [SnmpSpecType.TELEMETRY_QUERYING, ' Telemetry'], + [SnmpSpecType.CLIENT_ATTRIBUTES_QUERYING, 'Client attributes'], + [SnmpSpecType.SHARED_ATTRIBUTES_SETTING, 'Shared attributes'], + [SnmpSpecType.TO_DEVICE_RPC_REQUEST, 'RPC request'] +]); + +export interface SnmpCommunicationConfig { + spec: SnmpSpecType; + mappings: SnmpMapping[]; + queryingFrequencyMs?: number; +} + +export interface SnmpMapping { + oid: string; + key: string; + dataType: DataType; } export type DeviceProfileTransportConfigurations = DefaultDeviceProfileTransportConfiguration & @@ -332,7 +361,11 @@ export function createDeviceProfileTransportConfiguration(type: DeviceTransportT transportConfiguration = {...lwm2mTransportConfiguration, type: DeviceTransportType.LWM2M}; break; case DeviceTransportType.SNMP: - const snmpTransportConfiguration: SnmpDeviceProfileTransportConfiguration = {}; + const snmpTransportConfiguration: SnmpDeviceProfileTransportConfiguration = { + timeoutMs: 0, + retries: 0, + communicationConfigs: null + }; transportConfiguration = {...snmpTransportConfiguration, type: DeviceTransportType.SNMP}; break; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ec084b8a3a..95dc0c2d72 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1297,6 +1297,31 @@ "sw-update-recourse": "Software update CoAP recourse", "sw-update-recourse-required": "Software update CoAP recourse is required.", "config-json-tab": "Json Config Profile Device" + }, + "snmp": { + "add-communication-config": "Add communication config", + "add-mapping": "Add mapping", + "communication-configs": "Communication configs", + "data-key": "Data key", + "data-key-required": "Data key is required.", + "data-type": "Data type", + "data-type-required": "Data type is required.", + "oid": "OID", + "oid-pattern": "Invalid OID format.", + "oid-required": "OID is required.", + "please-add-communication-config": "Please add communication config", + "please-add-mapping-config": "Please add mapping config", + "querying-frequency": "Querying frequency, ms", + "querying-frequency-invalid-format": "Querying frequency must be a positive integer.", + "querying-frequency-required": "Querying frequency is required.", + "retries": "Retries", + "retries-invalid-format": "Retries must be a positive integer.", + "retries-required": "Retries is required.", + "scope": "Scope", + "scope-required": "Scope is required.", + "timeout-ms": "Timeout, ms", + "timeout-ms-invalid-format": "Timeout must be a positive integer.", + "timeout-ms-required": "Timeout is required." } }, "dialog": { From e4dd517edd1a0d471a4ea0aa3e687ef26184713e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 11 Jun 2021 12:36:03 +0300 Subject: [PATCH 66/76] cache: added cache executor service --- .../src/main/resources/thingsboard.yml | 1 + .../dao/cache/CacheExecutorService.java | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 54721ba2fa..ffdb783b12 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -332,6 +332,7 @@ actors: cache: # caffeine or redis type: "${CACHE_TYPE:caffeine}" + maximumPoolSize: "${CACHE_MAXIMUM_POOL_SIZE:16}" # max pool size to process futures that calls the external cache attributes: # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java new file mode 100644 index 0000000000..ca00e41d79 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java @@ -0,0 +1,48 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao.cache; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.AbstractListeningExecutor; + +@Component +public class CacheExecutorService extends AbstractListeningExecutor { + + @Value("${cache.maximumPoolSize}") + private int poolSize; + + @Override + protected int getThreadPollSize() { + return poolSize; + } + +} From c843b2f09e67a084e11a34d8c7f7f81075adc4b8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 11 Jun 2021 12:36:51 +0300 Subject: [PATCH 67/76] cache: CachedAttributesService replaced directExecutor with CacheExecutorService --- .../dao/attributes/CachedAttributesService.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 87ddfeee9a..92c75760e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -17,11 +17,9 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; @@ -32,6 +30,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.cache.CacheExecutorService; import org.thingsboard.server.dao.service.Validator; import java.util.ArrayList; @@ -45,7 +44,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @Service @@ -59,12 +57,15 @@ public class CachedAttributesService implements AttributesService { private final AttributesCacheWrapper cacheWrapper; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; + private final CacheExecutorService cacheExecutorService; public CachedAttributesService(AttributesDao attributesDao, AttributesCacheWrapper cacheWrapper, - StatsFactory statsFactory) { + StatsFactory statsFactory, + CacheExecutorService cacheExecutorService) { this.attributesDao = attributesDao; this.cacheWrapper = cacheWrapper; + this.cacheExecutorService = cacheExecutorService; this.hitCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "hit"); this.missCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "miss"); @@ -88,7 +89,7 @@ public class CachedAttributesService implements AttributesService { // TODO: think if it's a good idea to store 'empty' attributes cacheWrapper.put(attributeCacheKey, foundAttrKvEntry.orElse(null)); return foundAttrKvEntry; - }, MoreExecutors.directExecutor()); + }, cacheExecutorService); } } @@ -111,7 +112,7 @@ public class CachedAttributesService implements AttributesService { notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); ListenableFuture> result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); - return Futures.transform(result, foundInDbAttributes -> mergeDbAndCacheAttributes(entityId, scope, cachedAttributes, notFoundAttributeKeys, foundInDbAttributes), MoreExecutors.directExecutor()); + return Futures.transform(result, foundInDbAttributes -> mergeDbAndCacheAttributes(entityId, scope, cachedAttributes, notFoundAttributeKeys, foundInDbAttributes), cacheExecutorService); } @@ -169,7 +170,7 @@ public class CachedAttributesService implements AttributesService { // TODO: can do if (attributesCache.get() != null) attributesCache.put() instead, but will be more twice more requests to cache List attributeKeys = attributes.stream().map(KvEntry::getKey).collect(Collectors.toList()); - future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor()); + future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), cacheExecutorService); return future; } @@ -177,7 +178,7 @@ public class CachedAttributesService implements AttributesService { public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { validate(entityId, scope); ListenableFuture> future = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); - future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor()); + future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), cacheExecutorService); return future; } From be61b3c841328b7d0d4d5570512fefcd9d852b30 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 14 Jun 2021 14:39:00 +0300 Subject: [PATCH 68/76] cache: CachedAttributesService Executor choice depends on local or remote cache type. For local - directExecutor, for remote (IO calls) - dedicated threadPool to unblock any caller thread. Test added. --- .../attributes/CachedAttributesService.java | 1 + .../CachedAttributesServiceTest.java | 88 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 92c75760e2..d4e29f02de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.Cache; diff --git a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java new file mode 100644 index 0000000000..af980a6301 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java @@ -0,0 +1,88 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao.attributes; + +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.Test; +import org.thingsboard.server.dao.cache.CacheConfiguration; +import org.thingsboard.server.dao.cache.CacheExecutorService; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.Mockito.mock; + +public class CachedAttributesServiceTest { + + public static final String REDIS = "redis"; + + @Test + public void givenLocalCacheTypeName_whenEquals_thenOK() { + assertThat(CachedAttributesService.LOCAL_CACHE_TYPE, is("caffeine")); + } + + @Test + public void givenCacheType_whenGetExecutor_thenDirectExecutor() { + CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); + CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); + willCallRealMethod().given(cachedAttributesService).getExecutor(any(), any()); + + assertThat(cachedAttributesService.getExecutor(null, cacheExecutorService), is(MoreExecutors.directExecutor())); + + CacheConfiguration cacheConfiguration = new CacheConfiguration(); + cacheConfiguration.setType(null); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); + + cacheConfiguration.setType(""); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); + + cacheConfiguration.setType(CachedAttributesService.LOCAL_CACHE_TYPE); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); + + } + + @Test + public void givenCacheType_whenGetExecutor_thenReturnCacheExecutorService() { + CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); + CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); + willCallRealMethod().given(cachedAttributesService).getExecutor(any(CacheConfiguration.class), any(CacheExecutorService.class)); + + CacheConfiguration cacheConfiguration = new CacheConfiguration(); + cacheConfiguration.setType(REDIS); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); + + cacheConfiguration.setType("unknownCacheType"); + assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); + + } + +} \ No newline at end of file From 248e35cb249eff03b25e8b983701181191e0ada1 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 17 Jun 2021 17:05:55 +0300 Subject: [PATCH 69/76] UI: Fixed not updated view entities table, after updated values --- .../home/components/widget/lib/alarms-table-widget.component.ts | 1 + .../components/widget/lib/entities-table-widget.component.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index 138c56d307..390559bab3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -274,6 +274,7 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.updateTitle(true); this.alarmsDatasource.updateAlarms(); this.clearCache(); + this.ctx.detectChanges(); } public pageLinkSortDirection(): SortDirection { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index 8994051223..68db8a0dcc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -47,7 +47,6 @@ import { isDefined, isNumber, isObject, - isString, isUndefined } from '@core/utils'; import cssjs from '@core/css/css'; @@ -236,6 +235,7 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni this.updateTitle(true); this.entityDatasource.dataUpdated(); this.clearCache(); + this.ctx.detectChanges(); } public pageLinkSortDirection(): SortDirection { From be3e97d3e161933b93fdc6affc300e7dae7d0a15 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 18 Jun 2021 12:24:31 +0300 Subject: [PATCH 70/76] Improvements for CoAP --- .../lwm2m/NoSecLwM2MIntegrationTest.java | 2 +- .../lwm2m/X509LwM2MIntegrationTest.java | 7 +++- .../coapserver/DefaultCoapServerService.java | 13 ++++++- .../coap/OtaPackageTransportResource.java | 37 +++++++++++++++++++ .../uplink/DefaultLwM2MUplinkMsgHandler.java | 1 + 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java index e94f432f64..d98a775c8c 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/NoSecLwM2MIntegrationTest.java @@ -53,7 +53,7 @@ public class NoSecLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { private final int PORT = 5685; private final Security SECURITY = noSec("coap://localhost:" + PORT, 123); private final NetworkConfig COAP_CONFIG = new NetworkConfig().setString("COAP_PORT", Integer.toString(PORT)); - private final String ENDPOINT = "deviceAEndpoint"; + private final String ENDPOINT = "noSecEndpoint"; private Device createDevice() throws Exception { Device device = new Device(); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java index a5efff968a..661f7c5474 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/X509LwM2MIntegrationTest.java @@ -19,6 +19,7 @@ import org.eclipse.californium.core.network.config.NetworkConfig; import org.eclipse.leshan.client.object.Security; import org.jetbrains.annotations.NotNull; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; @@ -74,11 +75,13 @@ public class X509LwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { return device; } + //TODO: use different endpoints to isolate tests. + @Ignore() @Test public void testConnectAndObserveTelemetry() throws Exception { createDeviceProfile(TRANSPORT_CONFIGURATION); X509ClientCredentials credentials = new X509ClientCredentials(); - credentials.setEndpoint(endpoint); + credentials.setEndpoint(endpoint+1); Device device = createDevice(credentials); SingleEntityFilter sef = new SingleEntityFilter(); @@ -96,7 +99,7 @@ public class X509LwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { wsClient.waitForReply(); wsClient.registerWaitForUpdate(); - LwM2MTestClient client = new LwM2MTestClient(executor, endpoint); + LwM2MTestClient client = new LwM2MTestClient(executor, endpoint+1); Security security = x509(serverUri, 123, clientX509Cert.getEncoded(), clientPrivateKeyFromCert.getEncoded(), serverX509Cert.getEncoded()); client.init(security, coapConfig); String msg = wsClient.waitForUpdate(); diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java index 93f4e76551..605952e456 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java @@ -36,6 +36,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.eclipse.californium.core.network.config.NetworkConfigDefaults.DEFAULT_BLOCKWISE_STATUS_LIFETIME; + @Slf4j @Component @TbCoapServerComponent @@ -91,7 +93,16 @@ public class DefaultCoapServerService implements CoapServerService { InetAddress addr = InetAddress.getByName(coapServerContext.getHost()); InetSocketAddress sockAddr = new InetSocketAddress(addr, coapServerContext.getPort()); noSecCoapEndpointBuilder.setInetSocketAddress(sockAddr); - noSecCoapEndpointBuilder.setNetworkConfig(NetworkConfig.getStandard()); + NetworkConfig networkConfig = new NetworkConfig(); + networkConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_STRICT_BLOCK2_OPTION, true); + networkConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_ENTITY_TOO_LARGE_AUTO_FAILOVER, true); + networkConfig.setLong(NetworkConfig.Keys.BLOCKWISE_STATUS_LIFETIME, DEFAULT_BLOCKWISE_STATUS_LIFETIME); + networkConfig.setInt(NetworkConfig.Keys.MAX_RESOURCE_BODY_SIZE, 256 * 1024 * 1024); + networkConfig.setString(NetworkConfig.Keys.RESPONSE_MATCHING, "RELAXED"); + networkConfig.setInt(NetworkConfig.Keys.PREFERRED_BLOCK_SIZE, 1024); + networkConfig.setInt(NetworkConfig.Keys.MAX_MESSAGE_SIZE, 1024); + networkConfig.setInt(NetworkConfig.Keys.MAX_RETRANSMIT, 4); + noSecCoapEndpointBuilder.setNetworkConfig(networkConfig); CoapEndpoint noSecCoapEndpoint = noSecCoapEndpointBuilder.build(); server.addEndpoint(noSecCoapEndpoint); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java index cb403c5e82..bc9fdd9282 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/OtaPackageTransportResource.java @@ -20,8 +20,10 @@ import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.coap.Response; import org.eclipse.californium.core.network.Exchange; +import org.eclipse.californium.core.observe.ObserveRelation; import org.eclipse.californium.core.server.resources.CoapExchange; import org.eclipse.californium.core.server.resources.Resource; +import org.eclipse.californium.core.server.resources.ResourceObserver; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -41,6 +43,8 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { public OtaPackageTransportResource(CoapTransportContext ctx, OtaPackageType otaPackageType) { super(ctx, otaPackageType.getKeyPrefix()); + this.setObservable(true); + this.addObserver(new OtaPackageTransportResource.CoapResourceObserver()); this.otaPackageType = otaPackageType; } @@ -135,10 +139,43 @@ public class OtaPackageTransportResource extends AbstractCoapTransportResource { if (exchange.getRequestOptions().getBlock2() != null) { int chunkSize = exchange.getRequestOptions().getBlock2().getSzx(); boolean lastFlag = data.length > chunkSize; + response.getOptions().setUriPath(exchange.getRequestOptions().getUriPathString()); response.getOptions().setBlock2(chunkSize, lastFlag, 0); } exchange.respond(response); } } + public class CoapResourceObserver implements ResourceObserver { + @Override + public void changedName(String old) { + + } + + @Override + public void changedPath(String old) { + + } + + @Override + public void addedChild(Resource child) { + + } + + @Override + public void removedChild(Resource child) { + + } + + @Override + public void addedObserveRelation(ObserveRelation relation) { + + } + + @Override + public void removedObserveRelation(ObserveRelation relation) { + + } + } + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 5b61ffc768..cd9c311b54 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -256,6 +256,7 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl LwM2mClient lwM2MClient = clientContext.getClientByEndpoint(registration.getEndpoint()); try { log.warn("[{}] [{{}] Client: update after Registration", registration.getEndpoint(), registration.getId()); + logService.log(lwM2MClient, String.format("[%s][%s] Updated registration.", registration.getId(), registration.getSocketAddress())); clientContext.updateRegistration(lwM2MClient, registration); TransportProtos.SessionInfoProto sessionInfo = lwM2MClient.getSession(); this.reportActivityAndRegister(sessionInfo); From fe3378eab125b6f59ae316890f3ed4384b16f114 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 18 Jun 2021 12:58:38 +0300 Subject: [PATCH 71/76] UI: Add SNMP device transport configuration --- .../SnmpDeviceTransportConfiguration.java | 9 +- ...-profile-communication-config.component.ts | 4 +- ...snmp-device-profile-mapping.component.scss | 3 + ...ofile-transport-configuration.component.ts | 2 +- .../device/data/device-data.component.ts | 36 +++++- ...evice-transport-configuration.component.ts | 44 +++++-- ...ice-transport-configuration.component.html | 121 +++++++++++++++++- ...evice-transport-configuration.component.ts | 112 +++++++++++++--- ui-ngx/src/app/shared/models/device.models.ts | 60 ++++++++- .../assets/locale/locale.constant-en_US.json | 25 +++- 10 files changed, 366 insertions(+), 50 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index a7bc143d81..d377ede985 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -51,6 +51,13 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur private String privacyPassphrase; private String engineId; + public SnmpDeviceTransportConfiguration() { + this.host = "localhost"; + this.port = 161; + this.protocolVersion = SnmpProtocolVersion.V2C; + this.community = "public"; + } + @Override public DeviceTransportType getType() { return DeviceTransportType.SNMP; @@ -76,7 +83,7 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur isValid = StringUtils.isNotBlank(username) && StringUtils.isNotBlank(securityName) && contextName != null && authenticationProtocol != null && StringUtils.isNotBlank(authenticationPassphrase) - && privacyProtocol != null && privacyPassphrase != null && engineId != null; + && privacyProtocol != null && StringUtils.isNotBlank(privacyPassphrase) && engineId != null; break; } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts index d683d46880..7af2140efe 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-communication-config.component.ts @@ -179,7 +179,7 @@ export class SnmpDeviceProfileCommunicationConfigComponent implements OnInit, On if (isUndefinedOrNull(value)) { value = { spec: this.getFirstUnusedSeverity(), - queryingFrequencyMs: 0, + queryingFrequencyMs: 5000, mappings: null }; } @@ -196,7 +196,7 @@ export class SnmpDeviceProfileCommunicationConfigComponent implements OnInit, On ).subscribe(spec => { if (this.isShowFrequency(spec)) { form.addControl('queryingFrequencyMs', - this.fb.control(0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); + this.fb.control(5000, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')])); } else { form.removeControl('queryingFrequencyMs'); } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss index 94cf2290ac..5972aee722 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-mapping.component.scss @@ -40,6 +40,9 @@ .mat-form-field-underline { bottom: 0; } + .mat-form-field-subscript-wrapper{ + margin-top: 1.8em; + } } } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts index a384acbae3..400199ef9c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/snpm/snmp-device-profile-transport-configuration.component.ts @@ -86,7 +86,7 @@ export class SnmpDeviceProfileTransportConfigurationComponent implements OnInit, ngOnInit(): void { this.snmpDeviceProfileTransportConfigurationFormGroup = this.fb.group({ - timeoutMs: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + timeoutMs: [500, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], retries: [0, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], communicationConfigs: [null, Validators.required], }); diff --git a/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts b/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts index 1a510692db..a175764d89 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/data/device-data.component.ts @@ -15,7 +15,16 @@ /// import { Component, forwardRef, Input, OnInit } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; import { Store } from '@ngrx/store'; import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; @@ -29,13 +38,20 @@ import { selector: 'tb-device-data', templateUrl: './device-data.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => DeviceDataComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => DeviceDataComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => DeviceDataComponent), + multi: true + }, + ] }) -export class DeviceDataComponent implements ControlValueAccessor, OnInit { +export class DeviceDataComponent implements ControlValueAccessor, OnInit, Validator { deviceDataFormGroup: FormGroup; @@ -97,6 +113,12 @@ export class DeviceDataComponent implements ControlValueAccessor, OnInit { this.deviceDataFormGroup.patchValue({transportConfiguration: value?.transportConfiguration}, {emitEvent: false}); } + validate(): ValidationErrors | null { + return this.deviceDataFormGroup.valid ? null : { + deviceDataForm: false + }; + } + private updateModel() { let deviceData: DeviceData = null; if (this.deviceDataFormGroup.valid) { diff --git a/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts index 0e9d7e24a9..82359232a9 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.ts @@ -15,27 +15,39 @@ /// import { Component, forwardRef, Input, OnInit } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; import { Store } from '@ngrx/store'; import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; -import { - DeviceTransportConfiguration, - DeviceTransportType -} from '@shared/models/device.models'; +import { DeviceTransportConfiguration, DeviceTransportType } from '@shared/models/device.models'; import { deepClone } from '@core/utils'; @Component({ selector: 'tb-device-transport-configuration', templateUrl: './device-transport-configuration.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => DeviceTransportConfigurationComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => DeviceTransportConfigurationComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => DeviceTransportConfigurationComponent), + multi: true + }] }) -export class DeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit { +export class DeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit, Validator { deviceTransportType = DeviceTransportType; @@ -92,7 +104,15 @@ export class DeviceTransportConfigurationComponent implements ControlValueAccess if (configuration) { delete configuration.type; } - this.deviceTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + setTimeout(() => { + this.deviceTransportConfigurationFormGroup.patchValue({configuration}, {emitEvent: false}); + }, 0); + } + + validate(): ValidationErrors | null { + return this.deviceTransportConfigurationFormGroup.valid ? null : { + deviceTransportConfiguration: false + }; } private updateModel() { diff --git a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html index fc9f615db9..dadc98425e 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html @@ -15,10 +15,119 @@ limitations under the License. --> -
- - + +
+ + device-profile.snmp.host + + + {{ 'device-profile.snmp.host-required' | translate }} + + + + device-profile.snmp.port + + + {{ 'device-profile.snmp.port-required' | translate }} + + + {{ 'device-profile.snmp.port-format' | translate }} + + +
+ + device-profile.snmp.protocol-version + + + {{ snmpDeviceProtocolVersion | lowercase }} + + + + {{ 'device-profile.snmp.protocol-version-required' | translate }} + + +
+ + device-profile.snmp.community + + + {{ 'device-profile.snmp.community-required' | translate }} + + +
+
+
+ + device-profile.snmp.user-name + + + {{ 'device-profile.snmp.user-name-required' | translate }} + + + + device-profile.snmp.security-name + + + {{ 'device-profile.snmp.security-name-required' | translate }} + + +
+
+ + device-profile.snmp.authentication-protocol + + + {{ snmpAuthenticationProtocolTranslation.get(snmpAuthenticationProtocol) }} + + + + {{ 'device-profile.snmp.authentication-protocol-required' | translate }} + + + + device-profile.snmp.authentication-passphrase + + + {{ 'device-profile.snmp.authentication-passphrase-required' | translate }} + + +
+
+ + device-profile.snmp.privacy-protocol + + + {{ snmpPrivacyProtocolTranslation.get(snmpPrivacyProtocol) }} + + + + {{ 'device-profile.snmp.privacy-protocol-required' | translate }} + + + + device-profile.snmp.privacy-passphrase + + + {{ 'device-profile.snmp.privacy-passphrase-required' | translate }} + + +
+
+ + device-profile.snmp.context-name + + + + device-profile.snmp.engine-id + + +
+
diff --git a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts index 165a2906f4..f35d538bfa 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts @@ -14,31 +14,57 @@ /// limitations under the License. /// -import {Component, forwardRef, Input, OnInit} from '@angular/core'; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; -import {Store} from '@ngrx/store'; -import {AppState} from '@app/core/core.state'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { DeviceTransportConfiguration, DeviceTransportType, - SnmpDeviceTransportConfiguration + SnmpAuthenticationProtocol, + SnmpAuthenticationProtocolTranslationMap, + SnmpDeviceProtocolVersion, + SnmpDeviceTransportConfiguration, + SnmpPrivacyProtocol, + SnmpPrivacyProtocolTranslationMap } from '@shared/models/device.models'; +import { isDefinedAndNotNull } from '@core/utils'; @Component({ selector: 'tb-snmp-device-transport-configuration', templateUrl: './snmp-device-transport-configuration.component.html', styleUrls: [], - providers: [{ - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => SnmpDeviceTransportConfigurationComponent), - multi: true - }] + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceTransportConfigurationComponent), + multi: true + }, { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SnmpDeviceTransportConfigurationComponent), + multi: true + }] }) -export class SnmpDeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit { +export class SnmpDeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit, Validator { snmpDeviceTransportConfigurationFormGroup: FormGroup; + snmpDeviceProtocolVersions = Object.values(SnmpDeviceProtocolVersion); + snmpAuthenticationProtocols = Object.values(SnmpAuthenticationProtocol); + snmpAuthenticationProtocolTranslation = SnmpAuthenticationProtocolTranslationMap; + snmpPrivacyProtocols = Object.values(SnmpPrivacyProtocol); + snmpPrivacyProtocolTranslation = SnmpPrivacyProtocolTranslationMap; + private requiredValue: boolean; get required(): boolean { @@ -53,8 +79,7 @@ export class SnmpDeviceTransportConfigurationComponent implements ControlValueAc @Input() disabled: boolean; - private propagateChange = (v: any) => { - }; + private propagateChange = (v: any) => { }; constructor(private store: Store, private fb: FormBuilder) { @@ -69,13 +94,33 @@ export class SnmpDeviceTransportConfigurationComponent implements ControlValueAc ngOnInit() { this.snmpDeviceTransportConfigurationFormGroup = this.fb.group({ - configuration: [null, Validators.required] + host: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + port: [null, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], + protocolVersion: [SnmpDeviceProtocolVersion.V2C, Validators.required], + community: ['public', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + username: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + securityName: ['public', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + contextName: [null], + authenticationProtocol: [SnmpAuthenticationProtocol.SHA_512, Validators.required], + authenticationPassphrase: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + privacyProtocol: [SnmpPrivacyProtocol.DES, Validators.required], + privacyPassphrase: ['', [Validators.required, Validators.pattern('(.|\\s)*\\S(.|\\s)*')]], + engineId: [''] + }); + this.snmpDeviceTransportConfigurationFormGroup.get('protocolVersion').valueChanges.subscribe((protocol: SnmpDeviceProtocolVersion) => { + this.updateDisabledFormValue(protocol); }); this.snmpDeviceTransportConfigurationFormGroup.valueChanges.subscribe(() => { this.updateModel(); }); } + validate(): ValidationErrors | null { + return this.snmpDeviceTransportConfigurationFormGroup.valid ? null : { + snmpDeviceTransportConfiguration: false + }; + } + setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; if (this.disabled) { @@ -86,13 +131,46 @@ export class SnmpDeviceTransportConfigurationComponent implements ControlValueAc } writeValue(value: SnmpDeviceTransportConfiguration | null): void { - this.snmpDeviceTransportConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + if (isDefinedAndNotNull(value)) { + this.snmpDeviceTransportConfigurationFormGroup.patchValue(value, {emitEvent: false}); + if (this.snmpDeviceTransportConfigurationFormGroup.enabled) { + this.updateDisabledFormValue(value.protocolVersion || SnmpDeviceProtocolVersion.V2C); + } + } + } + + isV3protocolVersion(): boolean { + return this.snmpDeviceTransportConfigurationFormGroup.get('protocolVersion').value === SnmpDeviceProtocolVersion.V3; + } + + private updateDisabledFormValue(protocol: SnmpDeviceProtocolVersion) { + if (protocol === SnmpDeviceProtocolVersion.V3) { + this.snmpDeviceTransportConfigurationFormGroup.get('community').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('username').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('securityName').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('contextName').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationProtocol').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationPassphrase').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyProtocol').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyPassphrase').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('engineId').enable({emitEvent: false}); + } else { + this.snmpDeviceTransportConfigurationFormGroup.get('community').enable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('username').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('securityName').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('contextName').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationProtocol').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('authenticationPassphrase').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyProtocol').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('privacyPassphrase').disable({emitEvent: false}); + this.snmpDeviceTransportConfigurationFormGroup.get('engineId').disable({emitEvent: false}); + } } private updateModel() { let configuration: DeviceTransportConfiguration = null; if (this.snmpDeviceTransportConfigurationFormGroup.valid) { - configuration = this.snmpDeviceTransportConfigurationFormGroup.getRawValue().configuration; + configuration = this.snmpDeviceTransportConfigurationFormGroup.value; configuration.type = DeviceTransportType.SNMP; } this.propagateChange(configuration); diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 568faeb452..a766c3a6c3 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -362,7 +362,7 @@ export function createDeviceProfileTransportConfiguration(type: DeviceTransportT break; case DeviceTransportType.SNMP: const snmpTransportConfiguration: SnmpDeviceProfileTransportConfiguration = { - timeoutMs: 0, + timeoutMs: 500, retries: 0, communicationConfigs: null }; @@ -394,7 +394,12 @@ export function createDeviceTransportConfiguration(type: DeviceTransportType): D transportConfiguration = {...lwm2mTransportConfiguration, type: DeviceTransportType.LWM2M}; break; case DeviceTransportType.SNMP: - const snmpTransportConfiguration: SnmpDeviceTransportConfiguration = {}; + const snmpTransportConfiguration: SnmpDeviceTransportConfiguration = { + host: 'localhost', + port: 161, + protocolVersion: SnmpDeviceProtocolVersion.V2C, + community: 'public' + }; transportConfiguration = {...snmpTransportConfiguration, type: DeviceTransportType.SNMP}; break; } @@ -572,8 +577,57 @@ export interface Lwm2mDeviceTransportConfiguration { [key: string]: any; } +export enum SnmpDeviceProtocolVersion { + V1 = 'V1', + V2C = 'V2C', + V3 = 'V3' +} + +export enum SnmpAuthenticationProtocol { + SHA_1 = 'SHA_1', + SHA_224 = 'SHA_224', + SHA_256 = 'SHA_256', + SHA_384 = 'SHA_384', + SHA_512 = 'SHA_512', + MD5 = 'MD%' +} + +export const SnmpAuthenticationProtocolTranslationMap = new Map([ + [SnmpAuthenticationProtocol.SHA_1, 'SHA-1'], + [SnmpAuthenticationProtocol.SHA_224, 'SHA-224'], + [SnmpAuthenticationProtocol.SHA_256, 'SHA-256'], + [SnmpAuthenticationProtocol.SHA_384, 'SHA-384'], + [SnmpAuthenticationProtocol.SHA_512, 'SHA-512'], + [SnmpAuthenticationProtocol.MD5, 'MD5'] +]); + +export enum SnmpPrivacyProtocol { + DES = 'DES', + AES_128 = 'AES_128', + AES_192 = 'AES_192', + AES_256 = 'AES_256' +} + +export const SnmpPrivacyProtocolTranslationMap = new Map([ + [SnmpPrivacyProtocol.DES, 'DES'], + [SnmpPrivacyProtocol.AES_128, 'AES-128'], + [SnmpPrivacyProtocol.AES_192, 'AES-192'], + [SnmpPrivacyProtocol.AES_256, 'AES-256'], +]); + export interface SnmpDeviceTransportConfiguration { - [key: string]: any; + host?: string; + port?: number; + protocolVersion?: SnmpDeviceProtocolVersion; + community?: string; + username?: string; + securityName?: string; + contextName?: string; + authenticationProtocol?: SnmpAuthenticationProtocol; + authenticationPassphrase?: string; + privacyProtocol?: SnmpPrivacyProtocol; + privacyPassphrase?: string; + engineId?: string; } export type DeviceTransportConfigurations = DefaultDeviceTransportConfiguration & diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 95dc0c2d72..ff07be44da 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1301,16 +1301,35 @@ "snmp": { "add-communication-config": "Add communication config", "add-mapping": "Add mapping", + "authentication-passphrase": "Authentication passphrase", + "authentication-passphrase-required": "Authentication passphrase is required.", + "authentication-protocol": "Authentication protocol", + "authentication-protocol-required": "Authentication protocol is required.", "communication-configs": "Communication configs", + "community": "Community string", + "community-required": "Community string is required.", + "context-name": "Context name", "data-key": "Data key", "data-key-required": "Data key is required.", "data-type": "Data type", "data-type-required": "Data type is required.", + "engine-id": "Engine ID", + "host": "Host", + "host-required": "Host is required.", "oid": "OID", "oid-pattern": "Invalid OID format.", "oid-required": "OID is required.", "please-add-communication-config": "Please add communication config", "please-add-mapping-config": "Please add mapping config", + "port": "Port", + "port-format": "Invalid port format.", + "port-required": "Port is required.", + "privacy-passphrase": "Privacy passphrase", + "privacy-passphrase-required": "Privacy passphrase is required.", + "privacy-protocol": "Privacy protocol", + "privacy-protocol-required": "Privacy protocol is required.", + "protocol-version": "Protocol version", + "protocol-version-required": "Protocol version is required.", "querying-frequency": "Querying frequency, ms", "querying-frequency-invalid-format": "Querying frequency must be a positive integer.", "querying-frequency-required": "Querying frequency is required.", @@ -1319,9 +1338,13 @@ "retries-required": "Retries is required.", "scope": "Scope", "scope-required": "Scope is required.", + "security-name": "Security name", + "security-name-required": "Security name is required.", "timeout-ms": "Timeout, ms", "timeout-ms-invalid-format": "Timeout must be a positive integer.", - "timeout-ms-required": "Timeout is required." + "timeout-ms-required": "Timeout is required.", + "user-name": "User name", + "user-name-required": "User name is required." } }, "dialog": { From e3c523cc6454db8a6b10aa63b1c542444028a45e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 18 Jun 2021 14:33:31 +0300 Subject: [PATCH 72/76] Write replace command fix --- .../lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java | 2 +- .../server/transport/lwm2m/server/rpc/IdOrKeyRequest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java index 5de223fa2f..0432a5b742 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/DefaultLwM2MRpcRequestHandler.java @@ -206,7 +206,7 @@ public class DefaultLwM2MRpcRequestHandler implements LwM2MRpcRequestHandler { } private void sendWriteReplaceRequest(LwM2mClient client, TransportProtos.ToDeviceRpcRequestMsg requestMsg, String versionedId) { - RpcWriteUpdateRequest requestBody = JacksonUtil.fromString(requestMsg.getParams(), RpcWriteUpdateRequest.class); + RpcWriteReplaceRequest requestBody = JacksonUtil.fromString(requestMsg.getParams(), RpcWriteReplaceRequest.class); TbLwM2MWriteReplaceRequest request = TbLwM2MWriteReplaceRequest.builder().versionedId(versionedId) .value(requestBody.getValue()) .timeout(this.config.getTimeout()).build(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java index 1055b01232..77d8cd3ea1 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/rpc/IdOrKeyRequest.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.transport.lwm2m.server.rpc; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; @Data +@JsonIgnoreProperties(ignoreUnknown = true) public class IdOrKeyRequest { private String key; From e470a032a9f2fa12d949f6ed83df3ba7ef39e58b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 18 Jun 2021 17:21:35 +0300 Subject: [PATCH 73/76] improved countByDeviceProfileAndEmptyOtaPackage api --- .../server/controller/DeviceController.java | 10 ++++++---- .../java/org/thingsboard/rest/client/RestClient.java | 2 +- ui-ngx/src/app/core/http/ota-package.service.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index b13393515c..5c7df50d09 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -782,15 +782,17 @@ public class DeviceController extends BaseController { } @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/devices/count/{otaPackageType}", method = RequestMethod.GET) + @RequestMapping(value = "/devices/count/{otaPackageType}/{deviceProfileId}", method = RequestMethod.GET) @ResponseBody - public Long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(@PathVariable("otaPackageType") String otaPackageType, - @RequestParam String deviceProfileId) throws ThingsboardException { + public Long countByDeviceProfileAndEmptyOtaPackage(@PathVariable("otaPackageType") String otaPackageType, + @PathVariable("deviceProfileId") String deviceProfileId) throws ThingsboardException { checkParameter("OtaPackageType", otaPackageType); checkParameter("DeviceProfileId", deviceProfileId); try { return deviceService.countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage( - getCurrentUser().getTenantId(), new DeviceProfileId(UUID.fromString(deviceProfileId)), OtaPackageType.valueOf(otaPackageType)); + getTenantId(), + new DeviceProfileId(UUID.fromString(deviceProfileId)), + OtaPackageType.valueOf(otaPackageType)); } catch (Exception e) { throw handleException(e); } diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index b716935b2a..6fbf28f24e 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -1257,7 +1257,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { params.put("deviceProfileId", deviceProfileId.getId().toString()); return restTemplate.exchange( - baseURL + "/api/devices/count/{otaPackageType}?deviceProfileId={deviceProfileId}", + baseURL + "/api/devices/count/{otaPackageType}/{deviceProfileId}", HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference() { diff --git a/ui-ngx/src/app/core/http/ota-package.service.ts b/ui-ngx/src/app/core/http/ota-package.service.ts index 1f9a36882a..a806751825 100644 --- a/ui-ngx/src/app/core/http/ota-package.service.ts +++ b/ui-ngx/src/app/core/http/ota-package.service.ts @@ -133,7 +133,7 @@ export class OtaPackageService { } public countUpdateDeviceAfterChangePackage(type: OtaUpdateType, entityId: EntityId, config?: RequestConfig): Observable { - return this.http.get(`/api/devices/count/${type}?deviceProfileId=${entityId.id}`, defaultHttpOptionsFromConfig(config)); + return this.http.get(`/api/devices/count/${type}/${entityId.id}`, defaultHttpOptionsFromConfig(config)); } public confirmDialogUpdatePackage(entity: BaseData&OtaPagesIds, From 5b998717e79f7fcde249b9a0a3686a2c0f5be68e Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 18 Jun 2021 18:00:12 +0300 Subject: [PATCH 74/76] Remove invalid test --- .../CachedAttributesServiceTest.java | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java diff --git a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java deleted file mode 100644 index af980a6301..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL - * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. - * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. - * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. - */ -package org.thingsboard.server.dao.attributes; - -import com.google.common.util.concurrent.MoreExecutors; -import org.junit.Test; -import org.thingsboard.server.dao.cache.CacheConfiguration; -import org.thingsboard.server.dao.cache.CacheExecutorService; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.willCallRealMethod; -import static org.mockito.Mockito.mock; - -public class CachedAttributesServiceTest { - - public static final String REDIS = "redis"; - - @Test - public void givenLocalCacheTypeName_whenEquals_thenOK() { - assertThat(CachedAttributesService.LOCAL_CACHE_TYPE, is("caffeine")); - } - - @Test - public void givenCacheType_whenGetExecutor_thenDirectExecutor() { - CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); - CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); - willCallRealMethod().given(cachedAttributesService).getExecutor(any(), any()); - - assertThat(cachedAttributesService.getExecutor(null, cacheExecutorService), is(MoreExecutors.directExecutor())); - - CacheConfiguration cacheConfiguration = new CacheConfiguration(); - cacheConfiguration.setType(null); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); - - cacheConfiguration.setType(""); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); - - cacheConfiguration.setType(CachedAttributesService.LOCAL_CACHE_TYPE); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(MoreExecutors.directExecutor())); - - } - - @Test - public void givenCacheType_whenGetExecutor_thenReturnCacheExecutorService() { - CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); - CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); - willCallRealMethod().given(cachedAttributesService).getExecutor(any(CacheConfiguration.class), any(CacheExecutorService.class)); - - CacheConfiguration cacheConfiguration = new CacheConfiguration(); - cacheConfiguration.setType(REDIS); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); - - cacheConfiguration.setType("unknownCacheType"); - assertThat(cachedAttributesService.getExecutor(cacheConfiguration, cacheExecutorService), is(cacheExecutorService)); - - } - -} \ No newline at end of file From d64a0c382d7bac67224db60879ce640e8d3bd59a Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 18 Jun 2021 18:22:48 +0300 Subject: [PATCH 75/76] Fix test properties --- dao/src/test/resources/application-test.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index d7a960471b..a257c8fbce 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -10,6 +10,7 @@ audit-log.default_query_period=30 audit-log.sink.type=none cache.type=caffeine +cache.maximumPoolSize=16 #cache.type=redis caffeine.specs.relations.timeToLiveInMinutes=1440 From a7a7c801f2ea59f72cec1c92007fb9c0ad6d4804 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 18 Jun 2021 18:31:52 +0300 Subject: [PATCH 76/76] Fix license header --- .../dao/cache/CacheExecutorService.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java index ca00e41d79..9f0f54bb46 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cache/CacheExecutorService.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.server.dao.cache;