tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Refactor JS-Executor microservice to typescript

pull/6893/head
Vladyslav_Prykhodko 4 years ago
parent
3ef8ec6d02
commit
ed6e636918
27 changed files with 2648 additions and 1733 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 30
      msa/js-executor/api/httpServer.js
  2. 46
      msa/js-executor/api/httpServer.ts
  3. 90
      msa/js-executor/api/jsExecutor.js
  4. 69
      msa/js-executor/api/jsExecutor.models.ts
  5. 93
      msa/js-executor/api/jsExecutor.ts
  6. 344
      msa/js-executor/api/jsInvokeMessageProcessor.js
  7. 331
      msa/js-executor/api/jsInvokeMessageProcessor.ts
  8. 37
      msa/js-executor/api/utils.js
  9. 66
      msa/js-executor/api/utils.ts
  10. 57
      msa/js-executor/config/logger.ts
  11. 20
      msa/js-executor/package.json
  12. 34
      msa/js-executor/pom.xml
  13. 220
      msa/js-executor/queue/awsSqsTemplate.js
  14. 215
      msa/js-executor/queue/awsSqsTemplate.ts
  15. 296
      msa/js-executor/queue/kafkaTemplate.js
  16. 291
      msa/js-executor/queue/kafkaTemplate.ts
  17. 162
      msa/js-executor/queue/pubSubTemplate.js
  18. 175
      msa/js-executor/queue/pubSubTemplate.ts
  19. 21
      msa/js-executor/queue/queue.models.ts
  20. 181
      msa/js-executor/queue/rabbitmqTemplate.js
  21. 151
      msa/js-executor/queue/rabbitmqTemplate.ts
  22. 177
      msa/js-executor/queue/serviceBusTemplate.js
  23. 175
      msa/js-executor/queue/serviceBusTemplate.ts
  24. 55
      msa/js-executor/server.js
  25. 85
      msa/js-executor/server.ts
  26. 13
      msa/js-executor/tsconfig.json
  27. 947
      msa/js-executor/yarn.lock

30
msa/js-executor/api/httpServer.js
View File

@ -1,30 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const config = require('config'),
logger = require('../config/logger')._logger('httpServer'),
express = require('express');
const httpPort = Number(config.get('http_port'));
const app = express();
app.get('/livenessProbe', async (req, res) => {
const date = new Date();
const message = { now: date.toISOString() };
res.send(message);
})
app.listen(httpPort, () => logger.info(`Started http endpoint on port ${httpPort}. Please, use /livenessProbe !`))

46
msa/js-executor/api/httpServer.ts
View File

@ -0,0 +1,46 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import express from 'express';
import { _logger} from '../config/logger';
export class HttpServer {
private logger = _logger('httpServer');
private app = express();
private server;
constructor(httpPort: number) {
this.app.get('/livenessProbe', async (req, res) => {
const message = {
now: new Date().toISOString()
};
res.send(message);
})
this.server = this.app.listen(httpPort, () => {
this.logger.info('Started http endpoint on port %s. Please, use /livenessProbe !', httpPort);
}).on('error', (error) => {
this.logger.error(error);
});
}
stop() {
this.server.close(() => {
this.logger.info('Http server stop');
});
}
}

90
msa/js-executor/api/jsExecutor.js
View File

@ -1,90 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const vm = require('vm');
function JsExecutor(useSandbox) {
this.useSandbox = useSandbox;
}
JsExecutor.prototype.compileScript = function(code) {
if (this.useSandbox) {
return createScript(code);
} else {
return createFunction(code);
}
}
JsExecutor.prototype.executeScript = function(script, args, timeout) {
if (this.useSandbox) {
return invokeScript(script, args, timeout);
} else {
return invokeFunction(script, args);
}
}
function createScript(code) {
return new Promise((resolve, reject) => {
try {
code = "("+code+")(...args)";
var script = new vm.Script(code);
resolve(script);
} catch (err) {
reject(err);
}
});
}
function invokeScript(script, args, timeout) {
return new Promise((resolve, reject) => {
try {
var sandbox = Object.create(null);
sandbox.args = args;
var result = script.runInNewContext(sandbox, {timeout: timeout});
resolve(result);
} catch (err) {
reject(err);
}
});
}
function createFunction(code) {
return new Promise((resolve, reject) => {
try {
code = "return ("+code+")(...args)";
const parsingContext = vm.createContext({});
const func = vm.compileFunction(code, ['args'], {parsingContext: parsingContext});
resolve(func);
} catch (err) {
reject(err);
}
});
}
function invokeFunction(func, args) {
return new Promise((resolve, reject) => {
try {
var result = func(args);
resolve(result);
} catch (err) {
reject(err);
}
});
}
module.exports = JsExecutor;

69
msa/js-executor/api/jsExecutor.models.ts
View File

@ -0,0 +1,69 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
export interface TbMessage {
scriptIdMSB: string;
scriptIdLSB: string;
}
export interface RemoteJsRequest {
compileRequest?: JsCompileRequest;
invokeRequest?: JsInvokeRequest;
releaseRequest?: JsReleaseRequest;
}
export interface JsReleaseRequest extends TbMessage {
functionName: string;
}
export interface JsInvokeRequest extends TbMessage {
functionName: string;
scriptBody: string;
timeout: number;
args: string[];
}
export interface JsCompileRequest extends TbMessage {
functionName: string;
scriptBody: string;
}
export interface JsReleaseResponse extends TbMessage {
success: boolean;
}
export interface JsCompileResponse extends TbMessage {
success: boolean;
errorCode?: number;
errorDetails?: string;
}
export interface JsInvokeResponse {
success: boolean;
result: string;
errorCode?: number;
errorDetails?: string;
}
export interface RemoteJsResponse {
requestIdMSB: string;
requestIdLSB: string;
compileResponse?: JsCompileResponse;
invokeResponse?: JsInvokeResponse;
releaseResponse?: JsReleaseResponse;
}

93
msa/js-executor/api/jsExecutor.ts
View File

@ -0,0 +1,93 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import vm, { Script } from 'vm';
export type TbScript = Script | Function;
export class JsExecutor {
useSandbox: boolean;
constructor(useSandbox: boolean) {
this.useSandbox = useSandbox;
}
compileScript(code: string): Promise<TbScript> {
if (this.useSandbox) {
return this.createScript(code);
} else {
return this.createFunction(code);
}
}
executeScript(script: TbScript, args: string[], timeout?: number): Promise<any> {
if (this.useSandbox) {
return this.invokeScript(script as Script, args, timeout);
} else {
return this.invokeFunction(script as Function, args);
}
}
private createScript(code: string): Promise<Script> {
return new Promise((resolve, reject) => {
try {
code = "("+code+")(...args)";
const script = new vm.Script(code);
resolve(script);
} catch (err) {
reject(err);
}
});
}
private invokeScript(script: Script, args: string[], timeout: number | undefined): Promise<any> {
return new Promise((resolve, reject) => {
try {
const sandbox = Object.create(null);
sandbox.args = args;
const result = script.runInNewContext(sandbox, {timeout: timeout});
resolve(result);
} catch (err) {
reject(err);
}
});
}
private createFunction(code: string): Promise<Function> {
return new Promise((resolve, reject) => {
try {
code = "return ("+code+")(...args)";
const parsingContext = vm.createContext({});
const func = vm.compileFunction(code, ['args'], {parsingContext: parsingContext});
resolve(func);
} catch (err) {
reject(err);
}
});
}
private invokeFunction(func: Function, args: string[]): Promise<any> {
return new Promise((resolve, reject) => {
try {
const result = func(args);
resolve(result);
} catch (err) {
reject(err);
}
});
}
}

344
msa/js-executor/api/jsInvokeMessageProcessor.js
View File

@ -1,344 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const COMPILATION_ERROR = 0;
const RUNTIME_ERROR = 1;
const TIMEOUT_ERROR = 2;
const UNRECOGNIZED = -1;
const config = require('config'),
Long = require('long'),
logger = require('../config/logger')._logger('JsInvokeMessageProcessor'),
Utils = require('./utils'),
JsExecutor = require('./jsExecutor');
const statFrequency = Number(config.get('script.stat_print_frequency'));
const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency'));
const useSandbox = config.get('script.use_sandbox') === 'true';
const maxActiveScripts = Number(config.get('script.max_active_scripts'));
const slowQueryLogMs = Number(config.get('script.slow_query_log_ms'));
const slowQueryLogBody = config.get('script.slow_query_log_body') === 'true';
const { performance } = require('node:perf_hooks');
function JsInvokeMessageProcessor(producer) {
this.producer = producer;
this.executor = new JsExecutor(useSandbox);
this.scriptMap = new Map();
this.scriptIds = [];
this.executedScriptIdsCounter = [];
this.executedScriptsCounter = 0;
this.lastStatTime = performance.now();
this.compilationTime = 0;
}
JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function (message) {
var tStart = performance.now();
let requestId;
let responseTopic;
let expireTs;
let headers;
let request;
let buf;
try {
request = JSON.parse(Buffer.from(message.data).toString('utf8'));
headers = message.headers;
buf = Buffer.from(headers.data['requestId']);
requestId = Utils.UUIDFromBuffer(buf);
buf = Buffer.from(headers.data['responseTopic']);
responseTopic = buf.toString('utf8');
buf = Buffer.from(headers.data['expireTs']);
expireTs = Long.fromBytes(buf, false, false).toNumber();
const now = Date.now();
// if (logger.isDebugEnabled()) {
// logger.debug('expireTs is %s, buf is %s. Now is %s, ms to expire left %s', expireTs, buf.toString('hex'), now, expireTs - now)
// }
if (expireTs && expireTs <= now) {
if (logger.isDebugEnabled()) {
logger.debug('Message expired! expireTs is %s, buf is %s. Now is %s, ms to expire left %s', expireTs, buf.toString('hex'), now, expireTs - now)
}
return;
}
logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic);
if (request.compileRequest) {
this.processCompileRequest(requestId, responseTopic, headers, request.compileRequest);
} else if (request.invokeRequest) {
this.processInvokeRequest(requestId, responseTopic, headers, request.invokeRequest);
} else if (request.releaseRequest) {
this.processReleaseRequest(requestId, responseTopic, headers, request.releaseRequest);
} else {
logger.error('[%s] Unknown request received!', requestId);
}
} catch (err) {
logger.error('[%s] Failed to process request: %s', requestId, err.message);
logger.error(err.stack);
}
var tFinish = performance.now();
var tTook = tFinish - tStart;
if (tTook > slowQueryLogMs) {
let functionName;
if (request.invokeRequest) {
try {
buf = Buffer.from(request.invokeRequest['functionName']);
functionName = buf.toString('utf8');
} catch (err) {
logger.error('[%s] Failed to read functionName from message header: %s', requestId, err.message);
logger.error(err.stack);
}
}
logger.warn('[%s] SLOW PROCESSING [%s]ms, functionName [%s]', requestId, tTook, functionName);
if (slowQueryLogBody) {
logger.info('Slow request body: %s', JSON.stringify(request, null, 4))
}
}
}
JsInvokeMessageProcessor.prototype.processCompileRequest = function (requestId, responseTopic, headers, compileRequest) {
var scriptId = getScriptId(compileRequest);
logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId);
this.executor.compileScript(compileRequest.scriptBody).then(
(script) => {
this.cacheScript(scriptId, script);
var compileResponse = createCompileResponse(scriptId, true);
logger.debug('[%s] Sending success compile response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse);
},
(err) => {
var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err);
logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse);
}
);
}
JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, responseTopic, headers, invokeRequest) {
var scriptId = getScriptId(invokeRequest);
logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId);
this.executedScriptsCounter++;
if (this.executedScriptsCounter % statFrequency == 0) {
const nowMs = performance.now();
const msSinceLastStat = nowMs - this.lastStatTime;
const requestsPerSec = msSinceLastStat == 0 ? statFrequency : statFrequency / msSinceLastStat * 1000;
this.lastStatTime = nowMs;
logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s], compilation [%s]ms', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec, this.compilationTime);
this.compilationTime = 0;
}
if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) {
logger.info('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody);
}
this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then(
(script) => {
this.executor.executeScript(script, invokeRequest.args, invokeRequest.timeout).then(
(result) => {
var invokeResponse = createInvokeResponse(result, true);
logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
},
(err) => {
var errorCode;
if (err && isString(err.message) && err.message.includes('Script execution timed out')) {
errorCode = TIMEOUT_ERROR;
} else {
errorCode = RUNTIME_ERROR;
}
var invokeResponse = createInvokeResponse("", false, errorCode, err);
logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
}
)
},
(err) => {
var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err);
logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
}
);
}
JsInvokeMessageProcessor.prototype.processReleaseRequest = function (requestId, responseTopic, headers, releaseRequest) {
var scriptId = getScriptId(releaseRequest);
logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId);
if (this.scriptMap.has(scriptId)) {
var index = this.scriptIds.indexOf(scriptId);
if (index > -1) {
this.scriptIds.splice(index, 1);
this.executedScriptIdsCounter.splice(index, 1);
}
this.scriptMap.delete(scriptId);
}
var releaseResponse = createReleaseResponse(scriptId, true);
logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, undefined, releaseResponse);
}
JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, headers, scriptId, compileResponse, invokeResponse, releaseResponse) {
var tStartSending = performance.now();
var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse);
var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8');
logger.debug('[%s] Sending response to queue, scriptId: [%s]', requestId, scriptId);
this.producer.send(responseTopic, scriptId, rawResponse, headers).then(
() => {
logger.debug('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId);
},
(err) => {
if (err) {
logger.error('[%s] Failed to send response to queue: %s', requestId, err.message);
logger.error(err.stack);
}
}
);
}
JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scriptBody) {
var self = this;
return new Promise(function (resolve, reject) {
const script = self.scriptMap.get(scriptId);
if (script) {
incrementUseScriptId.call(self, scriptId);
resolve(script);
} else {
const startTime = performance.now();
self.executor.compileScript(scriptBody).then(
(compiledScript) => {
self.compilationTime += (performance.now() - startTime);
self.cacheScript(scriptId, compiledScript);
resolve(compiledScript);
},
(err) => {
self.compilationTime += (performance.now() - startTime);
reject(err);
}
);
}
});
}
JsInvokeMessageProcessor.prototype.cacheScript = function (scriptId, script) {
if (!this.scriptMap.has(scriptId)) {
this.scriptIds.push(scriptId);
this.executedScriptIdsCounter.push(0);
while (this.scriptIds.length > maxActiveScripts) {
logger.info('Active scripts count [%s] exceeds maximum limit [%s]', this.scriptIds.length, maxActiveScripts);
deleteMinUsedScript.apply(this);
}
}
this.scriptMap.set(scriptId, script);
logger.info("scriptMap size is [%s]", this.scriptMap.size);
}
function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) {
const requestIdBits = Utils.UUIDToBits(requestId);
return {
requestIdMSB: requestIdBits[0],
requestIdLSB: requestIdBits[1],
compileResponse: compileResponse,
invokeResponse: invokeResponse,
releaseResponse: releaseResponse
};
}
function createCompileResponse(scriptId, success, errorCode, err) {
const scriptIdBits = Utils.UUIDToBits(scriptId);
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
};
}
function createInvokeResponse(result, success, errorCode, err) {
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
result: result
};
}
function createReleaseResponse(scriptId, success) {
const scriptIdBits = Utils.UUIDToBits(scriptId);
return {
success: success,
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
};
}
function parseJsErrorDetails(err) {
if (!err) {
return undefined;
}
var details = err.name + ': ' + err.message;
if (err.stack) {
var lines = err.stack.split('\n');
if (lines && lines.length) {
var line = lines[0];
var splitted = line.split(':');
if (splitted && splitted.length === 2) {
if (!isNaN(splitted[1])) {
details += ' in at line number ' + splitted[1];
}
}
}
}
return details;
}
function getScriptId(request) {
return Utils.toUUIDString(request.scriptIdMSB, request.scriptIdLSB);
}
function incrementUseScriptId(scriptId) {
const index = this.scriptIds.indexOf(scriptId);
if (this.executedScriptIdsCounter[index] < Number.MAX_SAFE_INTEGER) {
this.executedScriptIdsCounter[index]++;
}
}
function deleteMinUsedScript() {
let min = Infinity;
let minIndex = 0;
const scriptIdsLength = this.executedScriptIdsCounter.length - 1; // ignored last added script
for (let i = 0; i < scriptIdsLength; i++) {
if (this.executedScriptIdsCounter[i] < min) {
min = this.executedScriptIdsCounter[i];
minIndex = i;
}
}
const prevScriptId = this.scriptIds.splice(minIndex, 1)[0];
this.executedScriptIdsCounter.splice(minIndex, 1)
logger.info('Removing active script with id [%s]', prevScriptId);
this.scriptMap.delete(prevScriptId);
}
function isString(value) {
return typeof value === 'string';
}
module.exports = JsInvokeMessageProcessor;

331
msa/js-executor/api/jsInvokeMessageProcessor.ts
View File

@ -0,0 +1,331 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import { _logger } from '../config/logger';
import { JsExecutor, TbScript } from './jsExecutor';
import { performance } from 'perf_hooks';
import { isString, parseJsErrorDetails, toUUIDString, UUIDFromBuffer, UUIDToBits } from './utils';
import { IQueue } from '../queue/queue.models';
import {
JsCompileRequest,
JsCompileResponse,
JsInvokeRequest,
JsInvokeResponse,
JsReleaseRequest,
JsReleaseResponse,
RemoteJsRequest,
RemoteJsResponse,
TbMessage
} from './jsExecutor.models';
import Long from 'long';
const COMPILATION_ERROR = 0;
const RUNTIME_ERROR = 1;
const TIMEOUT_ERROR = 2;
const statFrequency = Number(config.get('script.stat_print_frequency'));
const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency'));
const useSandbox = config.get('script.use_sandbox') === 'true';
const maxActiveScripts = Number(config.get('script.max_active_scripts'));
const slowQueryLogMs = Number(config.get('script.slow_query_log_ms'));
const slowQueryLogBody = config.get('script.slow_query_log_body') === 'true';
export class JsInvokeMessageProcessor {
private logger = _logger(`JsInvokeMessageProcessor`);
private producer: IQueue;
private executor = new JsExecutor(useSandbox);
private scriptMap = new Map<string, TbScript>();
private scriptIds: string[] = [];
private executedScriptIdsCounter: number[] = [];
private executedScriptsCounter = 0;
private lastStatTime = performance.now();
private compilationTime = 0;
constructor(produced: IQueue) {
this.producer = produced;
}
onJsInvokeMessage(message: any) {
const tStart = performance.now();
let requestId = '';
let responseTopic: string;
let expireTs;
let headers;
let request: RemoteJsRequest = {};
let buf: Buffer;
try {
request = JSON.parse(Buffer.from(message.data).toString('utf8'));
headers = message.headers;
buf = Buffer.from(headers.data['requestId']);
requestId = UUIDFromBuffer(buf);
buf = Buffer.from(headers.data['responseTopic']);
responseTopic = buf.toString('utf8');
buf = Buffer.from(headers.data['expireTs']);
expireTs = Long.fromBytes(Array.from(buf), false, false).toNumber();
const now = Date.now();
if (expireTs && expireTs <= now) {
if (this.logger.isDebugEnabled()) {
this.logger.debug('Message expired! expireTs is %s, buf is %s. Now is %s, ms to expire left %s', expireTs, buf.toString('hex'), now, expireTs - now)
}
return;
}
this.logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic);
if (request.compileRequest) {
this.processCompileRequest(requestId, responseTopic, headers, request.compileRequest);
} else if (request.invokeRequest) {
this.processInvokeRequest(requestId, responseTopic, headers, request.invokeRequest);
} else if (request.releaseRequest) {
this.processReleaseRequest(requestId, responseTopic, headers, request.releaseRequest);
} else {
this.logger.error('[%s] Unknown request received!', requestId);
}
} catch (err: any) {
this.logger.error('[%s] Failed to process request: %s', requestId, err.message);
this.logger.error(err.stack);
}
const tFinish = performance.now();
const tTook = tFinish - tStart;
if (tTook > slowQueryLogMs) {
let functionName;
if (request.invokeRequest) {
try {
buf = Buffer.from(request.invokeRequest['functionName']);
functionName = buf.toString('utf8');
} catch (err: any) {
this.logger.error('[%s] Failed to read functionName from message header: %s', requestId, err.message);
this.logger.error(err.stack);
}
}
this.logger.warn('[%s] SLOW PROCESSING [%s]ms, functionName [%s]', requestId, tTook, functionName);
if (slowQueryLogBody) {
this.logger.info('Slow request body: %s', JSON.stringify(request, null, 4))
}
}
}
processCompileRequest(requestId: string, responseTopic: string, headers: any, compileRequest: JsCompileRequest) {
const scriptId = JsInvokeMessageProcessor.getScriptId(compileRequest);
this.logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId);
this.executor.compileScript(compileRequest.scriptBody).then(
(script) => {
this.cacheScript(scriptId, script);
const compileResponse = JsInvokeMessageProcessor.createCompileResponse(scriptId, true);
this.logger.debug('[%s] Sending success compile response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse);
},
(err) => {
const compileResponse = JsInvokeMessageProcessor.createCompileResponse(scriptId, false, COMPILATION_ERROR, err);
this.logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse);
}
);
}
processInvokeRequest(requestId: string, responseTopic: string, headers: any, invokeRequest: JsInvokeRequest) {
const scriptId = JsInvokeMessageProcessor.getScriptId(invokeRequest);
this.logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId);
this.executedScriptsCounter++;
if (this.executedScriptsCounter % statFrequency == 0) {
const nowMs = performance.now();
const msSinceLastStat = nowMs - this.lastStatTime;
const requestsPerSec = msSinceLastStat == 0 ? statFrequency : statFrequency / msSinceLastStat * 1000;
this.lastStatTime = nowMs;
this.logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s], compilation [%s]ms', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec, this.compilationTime);
this.compilationTime = 0;
}
if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) {
this.logger.info('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody);
}
this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then(
(script) => {
this.executor.executeScript(script, invokeRequest.args, invokeRequest.timeout).then(
(result) => {
const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse(result, true);
this.logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
},
(err: any) => {
let errorCode;
if (err && isString(err.message) && err.message.includes('Script execution timed out')) {
errorCode = TIMEOUT_ERROR;
} else {
errorCode = RUNTIME_ERROR;
}
const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse("", false, errorCode, err);
this.logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
}
)
},
(err: any) => {
const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse("", false, COMPILATION_ERROR, err);
this.logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
}
);
}
processReleaseRequest(requestId: string, responseTopic: string, headers: any, releaseRequest: JsReleaseRequest) {
const scriptId = JsInvokeMessageProcessor.getScriptId(releaseRequest);
this.logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId);
if (this.scriptMap.has(scriptId)) {
const index = this.scriptIds.indexOf(scriptId);
if (index > -1) {
this.scriptIds.splice(index, 1);
this.executedScriptIdsCounter.splice(index, 1);
}
this.scriptMap.delete(scriptId);
}
const releaseResponse = JsInvokeMessageProcessor.createReleaseResponse(scriptId, true);
this.logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, undefined, releaseResponse);
}
sendResponse(requestId: string, responseTopic: string, headers: any, scriptId: string,
compileResponse?: JsCompileResponse, invokeResponse?: JsInvokeResponse, releaseResponse?: JsReleaseResponse) {
const tStartSending = performance.now();
const remoteResponse = JsInvokeMessageProcessor.createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse);
const rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8');
this.logger.debug('[%s] Sending response to queue, scriptId: [%s]', requestId, scriptId);
this.producer.send(responseTopic, scriptId, rawResponse, headers).then(
() => {
this.logger.debug('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId);
},
(err: any) => {
if (err) {
this.logger.error('[%s] Failed to send response to queue: %s', requestId, err.message);
this.logger.error(err.stack);
}
}
);
}
getOrCompileScript(scriptId: string, scriptBody: string): Promise<TbScript> {
const self = this;
return new Promise(function (resolve, reject) {
const script = self.scriptMap.get(scriptId);
if (script) {
self.incrementUseScriptId(scriptId);
resolve(script);
} else {
const startTime = performance.now();
self.executor.compileScript(scriptBody).then(
(compiledScript) => {
self.compilationTime += (performance.now() - startTime);
self.cacheScript(scriptId, compiledScript);
resolve(compiledScript);
},
(err) => {
self.compilationTime += (performance.now() - startTime);
reject(err);
}
);
}
});
}
cacheScript(scriptId: string, script: TbScript) {
if (!this.scriptMap.has(scriptId)) {
this.scriptIds.push(scriptId);
this.executedScriptIdsCounter.push(0);
while (this.scriptIds.length > maxActiveScripts) {
this.logger.info('Active scripts count [%s] exceeds maximum limit [%s]', this.scriptIds.length, maxActiveScripts);
this.deleteMinUsedScript();
}
}
this.scriptMap.set(scriptId, script);
this.logger.info("scriptMap size is [%s]", this.scriptMap.size);
}
private static createRemoteResponse(requestId: string, compileResponse?: JsCompileResponse,
invokeResponse?: JsInvokeResponse, releaseResponse?: JsReleaseResponse): RemoteJsResponse {
const requestIdBits = UUIDToBits(requestId);
return {
requestIdMSB: requestIdBits[0],
requestIdLSB: requestIdBits[1],
compileResponse: compileResponse,
invokeResponse: invokeResponse,
releaseResponse: releaseResponse
};
}
private static createCompileResponse(scriptId: string, success: boolean, errorCode?: number, err?: any): JsCompileResponse {
const scriptIdBits = UUIDToBits(scriptId);
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
};
}
private static createInvokeResponse(result: string, success: boolean, errorCode?: number, err?: any): JsInvokeResponse {
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
result: result
};
}
private static createReleaseResponse(scriptId: string, success: boolean): JsReleaseResponse {
const scriptIdBits = UUIDToBits(scriptId);
return {
success: success,
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
};
}
private static getScriptId(request: TbMessage): string {
return toUUIDString(request.scriptIdMSB, request.scriptIdLSB);
}
private incrementUseScriptId(scriptId: string) {
const index = this.scriptIds.indexOf(scriptId);
if (this.executedScriptIdsCounter[index] < Number.MAX_SAFE_INTEGER) {
this.executedScriptIdsCounter[index]++;
}
}
private deleteMinUsedScript() {
let min = Infinity;
let minIndex = 0;
const scriptIdsLength = this.executedScriptIdsCounter.length - 1; // ignored last added script
for (let i = 0; i < scriptIdsLength; i++) {
if (this.executedScriptIdsCounter[i] < min) {
min = this.executedScriptIdsCounter[i];
minIndex = i;
}
}
const prevScriptId = this.scriptIds.splice(minIndex, 1)[0];
this.executedScriptIdsCounter.splice(minIndex, 1)
this.logger.info('Removing active script with id [%s]', prevScriptId);
this.scriptMap.delete(prevScriptId);
}
}

37
msa/js-executor/api/utils.js
View File

@ -1,37 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const Long = require('long'),
uuidParse = require('uuid-parse');
exports.toUUIDString = function(mostSigBits, leastSigBits) {
var msbBytes = Long.fromValue(mostSigBits, false).toBytes(false);
var lsbBytes = Long.fromValue(leastSigBits, false).toBytes(false);
var uuidBytes = msbBytes.concat(lsbBytes);
return uuidParse.unparse(uuidBytes);
}
exports.UUIDFromBuffer = function(buf) {
return uuidParse.unparse(buf);
}
exports.UUIDToBits = function(uuidString) {
const bytes = uuidParse.parse(uuidString);
var msb = Long.fromBytes(bytes.slice(0,8), false, false).toString();
var lsb = Long.fromBytes(bytes.slice(-8), false, false).toString();
return [msb, lsb];
}

66
msa/js-executor/api/utils.ts
View File

@ -0,0 +1,66 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import Long from 'long';
import uuidParse from 'uuid-parse';
export function toUUIDString(mostSigBits: string, leastSigBits: string): string {
const msbBytes = Long.fromValue(mostSigBits, false).toBytes(false);
const lsbBytes = Long.fromValue(leastSigBits, false).toBytes(false);
const uuidBytes = msbBytes.concat(lsbBytes);
return uuidParse.unparse(uuidBytes as any);
}
export function UUIDFromBuffer(buf: Buffer): string {
return uuidParse.unparse(buf);
}
export function UUIDToBits(uuidString: string): [string, string] {
const bytes = Array.from(uuidParse.parse(uuidString));
const msb = Long.fromBytes(bytes.slice(0, 8), false, false).toString();
const lsb = Long.fromBytes(bytes.slice(-8), false, false).toString();
return [msb, lsb];
}
export function isString(value: any): boolean {
return typeof value === 'string';
}
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
export function parseJsErrorDetails(err: any): string | undefined {
if (!err) {
return undefined;
}
let details = err.name + ': ' + err.message;
if (err.stack) {
const lines = err.stack.split('\n');
if (lines && lines.length) {
const line = lines[0];
const split = line.split(':');
if (split && split.length === 2) {
if (!isNaN(split[1])) {
details += ' in at line number ' + split[1];
}
}
}
}
return details;
}

57
msa/js-executor/config/logger.js → msa/js-executor/config/logger.ts
View File

@ -1,27 +1,30 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var config = require('config'),
path = require('path'),
DailyRotateFile = require('winston-daily-rotate-file');
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import path from 'path';
import DailyRotateFile from 'winston-daily-rotate-file';
import { LogEntry, logLevel } from 'kafkajs';
import { createLogger, format, transports } from 'winston';
import * as Transport from 'winston-transport';
const { logLevel } = require('kafkajs');
const { createLogger, format, transports } = require('winston');
const { combine, timestamp, label, printf, splat } = format;
const toWinstonLogLevel = level => {
const toWinstonLogLevel = (level: logLevel): string => {
switch(level) {
case logLevel.ERROR:
case logLevel.NOTHING:
@ -35,15 +38,15 @@ const toWinstonLogLevel = level => {
}
}
var loggerTransports = [];
const loggerTransports: Array<Transport> = [];
if (process.env.NODE_ENV !== 'production' || process.env.DOCKER_MODE === 'true') {
loggerTransports.push(new transports.Console({
handleExceptions: true
}));
} else {
var filename = path.join(config.get('logger.path'), config.get('logger.filename'));
var transport = new (DailyRotateFile)({
const filename = path.join(config.get('logger.path'), config.get('logger.filename'));
const transport = new (DailyRotateFile)({
filename: filename,
datePattern: 'YYYY-MM-DD-HH',
zippedArchive: true,
@ -58,7 +61,7 @@ const tbFormat = printf(info => {
return `${info.timestamp} [${info.label}] ${info.level.toUpperCase()}: ${info.message}`;
});
function _logger(moduleLabel) {
export function _logger(moduleLabel: string) {
return createLogger({
level: config.get('logger.level'),
format:combine(
@ -71,7 +74,7 @@ function _logger(moduleLabel) {
});
}
const KafkaJsWinstonLogCreator = logLevel => {
export function KafkaJsWinstonLogCreator(logLevel: logLevel): (entry: LogEntry) => void {
const logger = createLogger({
level: toWinstonLogLevel(logLevel),
format:combine(
@ -99,5 +102,3 @@ const KafkaJsWinstonLogCreator = logLevel => {
});
}
}
module.exports = {_logger, KafkaJsWinstonLogCreator};

20
msa/js-executor/package.json
View File

@ -3,19 +3,20 @@
"private": true,
"version": "3.4.0",
"description": "ThingsBoard JavaScript Executor Microservice",
"main": "server.js",
"main": "server.ts",
"bin": "server.js",
"scripts": {
"pkg": "pkg -t node16-linux-x64,node16-win-x64 --out-path ./target . && node install.js",
"pkg": "tsc && pkg -t node16-linux-x64,node16-win-x64 --out-path ./target ./target/src && node install.js",
"test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon server.js",
"start-prod": "NODE_ENV=production nodemon server.js"
"start": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web ts-node server.ts'",
"start-prod": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web NODE_ENV=production ts-node server.ts'",
"build": "tsc"
},
"dependencies": {
"@aws-sdk/client-sqs": "^3.121.0",
"@azure/service-bus": "^7.5.1",
"@google-cloud/pubsub": "^3.0.1",
"amqplib": "^0.10.0",
"aws-sdk": "^2.1152.0",
"config": "^3.3.7",
"express": "^4.18.1",
"js-yaml": "^4.1.0",
@ -35,9 +36,16 @@
]
},
"devDependencies": {
"@types/amqplib": "^0.8.2",
"@types/config": "^0.0.41",
"@types/express": "^4.17.13",
"@types/node": "^18.0.1",
"@types/uuid-parse": "^1.0.0",
"fs-extra": "^10.1.0",
"nodemon": "^2.0.16",
"pkg": "^5.7.0"
"pkg": "^5.7.0",
"ts-node": "^10.8.1",
"typescript": "^4.7.4"
},
"pkg": {
"assets": [

34
msa/js-executor/pom.xml
View File

@ -89,6 +89,7 @@
<goals>
<goal>yarn</goal>
</goals>
<phase>compile</phase>
<configuration>
<arguments>run pkg</arguments>
</configuration>
@ -103,38 +104,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-src-api</id>
<phase>process-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/src/api</outputDirectory>
<resources>
<resource>
<directory>api</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy-src-queue</id>
<phase>process-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/src/queue</outputDirectory>
<resources>
<resource>
<directory>queue</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy-src-root</id>
<phase>process-resources</phase>
@ -147,7 +116,6 @@
<resource>
<directory>${basedir}</directory>
<includes>
<include>server.js</include>
<include>package.json</include>
<include>yarn.lock</include>
</includes>

220
msa/js-executor/queue/awsSqsTemplate.js
View File

@ -1,220 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const config = require('config'),
JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
logger = require('../config/logger')._logger('awsSqsTemplate');
const uuid = require('uuid-random');
const requestTopic = config.get('request_topic');
const accessKeyId = config.get('aws_sqs.access_key_id');
const secretAccessKey = config.get('aws_sqs.secret_access_key');
const region = config.get('aws_sqs.region');
const AWS = require('aws-sdk');
const queueProperties = config.get('aws_sqs.queue_properties');
const pollInterval = config.get('js.response_poll_interval');
let queueAttributes = {FifoQueue: 'true'};
let sqsClient;
let requestQueueURL;
const queueUrls = new Map();
let stopped = false;
function AwsSqsProducer() {
this.send = async (responseTopic, scriptId, rawResponse, headers) => {
let msgBody = JSON.stringify(
{
key: scriptId,
data: [...rawResponse],
headers: headers
});
let responseQueueUrl = queueUrls.get(topicToSqsQueueName(responseTopic));
if (!responseQueueUrl) {
responseQueueUrl = await createQueue(responseTopic);
queueUrls.set(responseTopic, responseQueueUrl);
}
let msgId = uuid();
let params = {
MessageBody: msgBody,
QueueUrl: responseQueueUrl,
MessageGroupId: msgId,
MessageDeduplicationId: msgId
};
return new Promise((resolve, reject) => {
sqsClient.sendMessage(params, function (err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
}
(async () => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
AWS.config.update({accessKeyId: accessKeyId, secretAccessKey: secretAccessKey, region: region});
sqsClient = new AWS.SQS({apiVersion: '2012-11-05'});
const queues = await getQueues();
if (queues) {
queues.forEach(queueUrl => {
const delimiterPosition = queueUrl.lastIndexOf('/');
const queueName = queueUrl.substring(delimiterPosition + 1);
queueUrls.set(queueName, queueUrl);
});
}
parseQueueProperties();
requestQueueURL = queueUrls.get(topicToSqsQueueName(requestTopic));
if (!requestQueueURL) {
requestQueueURL = await createQueue(requestTopic);
}
const messageProcessor = new JsInvokeMessageProcessor(new AwsSqsProducer());
const params = {
MaxNumberOfMessages: 10,
QueueUrl: requestQueueURL,
WaitTimeSeconds: pollInterval / 1000
};
while (!stopped) {
let pollStartTs = new Date().getTime();
const messages = await new Promise((resolve, reject) => {
sqsClient.receiveMessage(params, function (err, data) {
if (err) {
reject(err);
} else {
resolve(data.Messages);
}
});
});
if (messages && messages.length > 0) {
const entries = [];
messages.forEach(message => {
entries.push({
Id: message.MessageId,
ReceiptHandle: message.ReceiptHandle
});
messageProcessor.onJsInvokeMessage(JSON.parse(message.Body));
});
const deleteBatch = {
QueueUrl: requestQueueURL,
Entries: entries
};
sqsClient.deleteMessageBatch(deleteBatch, function (err, data) {
if (err) {
logger.error("Failed to delete messages from queue.", err.message);
} else {
//do nothing
}
});
} else {
let pollDuration = new Date().getTime() - pollStartTs;
if (pollDuration < pollInterval) {
await sleep(pollInterval - pollDuration);
}
}
}
} catch (e) {
logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
logger.error(e.stack);
exit(-1);
}
})();
function createQueue(topic) {
let queueName = topicToSqsQueueName(topic);
let queueParams = {QueueName: queueName, Attributes: queueAttributes};
return new Promise((resolve, reject) => {
sqsClient.createQueue(queueParams, function (err, data) {
if (err) {
reject(err);
} else {
resolve(data.QueueUrl);
}
});
});
}
function getQueues() {
return new Promise((resolve, reject) => {
sqsClient.listQueues(function (err, data) {
if (err) {
reject(err);
} else {
resolve(data.QueueUrls);
}
});
});
}
function topicToSqsQueueName(topic) {
return topic.replace(/\./g, '_') + '.fifo';
}
function parseQueueProperties() {
const props = queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
});
}
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
process.on('exit', () => {
stopped = true;
logger.info('Aws Sqs client stopped.');
exit(0);
});
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
if (sqsClient) {
logger.info('Stopping Aws Sqs client.')
try {
await sqsClient.close();
logger.info('Aws Sqs client stopped.')
process.exit(status);
} catch (e) {
logger.info('Aws Sqs client stop error.');
process.exit(status);
}
} else {
process.exit(status);
}
}

215
msa/js-executor/queue/awsSqsTemplate.ts
View File

@ -0,0 +1,215 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import { _logger } from '../config/logger';
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
import { IQueue } from './queue.models';
import {
CreateQueueCommand,
CreateQueueRequest,
DeleteMessageBatchCommand,
DeleteMessageBatchRequest,
DeleteMessageBatchRequestEntry,
ListQueuesCommand,
ListQueuesResult,
ReceiveMessageCommand,
ReceiveMessageRequest,
ReceiveMessageResult,
SendMessageCommand,
SendMessageRequest,
SQSClient
} from '@aws-sdk/client-sqs';
import uuid from 'uuid-random';
import { sleep } from '../api/utils';
export class AwsSqsTemplate implements IQueue {
private logger = _logger(`awsSqsTemplate`);
private requestTopic: string = config.get('request_topic');
private accessKeyId: string = config.get('aws_sqs.access_key_id');
private secretAccessKey: string = config.get('aws_sqs.secret_access_key');
private region: string = config.get('aws_sqs.region');
private queueProperties: string = config.get('aws_sqs.queue_properties');
private pollInterval = Number(config.get('js.response_poll_interval'));
private sqsClient: SQSClient;
private requestQueueURL: string
private stopped = false;
private queueUrls = new Map<string, string>();
private queueAttributes: { [n: string]: string } = {
FifoQueue: 'true'
};
constructor() {
}
async init() {
try {
this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
this.sqsClient = new SQSClient({
apiVersion: '2012-11-05',
credentials: {
accessKeyId: this.accessKeyId,
secretAccessKey: this.secretAccessKey
},
region: this.region
});
const queues = await this.getQueues();
if (queues.QueueUrls) {
queues.QueueUrls.forEach(queueUrl => {
const delimiterPosition = queueUrl.lastIndexOf('/');
const queueName = queueUrl.substring(delimiterPosition + 1);
this.queueUrls.set(queueName, queueUrl);
});
}
this.parseQueueProperties();
this.requestQueueURL = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(this.requestTopic)) || '';
if (!this.requestQueueURL) {
this.requestQueueURL = await this.createQueue(this.requestTopic);
}
const messageProcessor = new JsInvokeMessageProcessor(this);
const params: ReceiveMessageRequest = {
MaxNumberOfMessages: 10,
QueueUrl: this.requestQueueURL,
WaitTimeSeconds: this.pollInterval / 1000
};
while (!this.stopped) {
let pollStartTs = new Date().getTime();
const messagesResponse: ReceiveMessageResult = await this.sqsClient.send(new ReceiveMessageCommand(params));
const messages = messagesResponse.Messages;
if (messages && messages.length > 0) {
const entries: DeleteMessageBatchRequestEntry[] = [];
messages.forEach(message => {
entries.push({
Id: message.MessageId,
ReceiptHandle: message.ReceiptHandle
});
messageProcessor.onJsInvokeMessage(JSON.parse(message.Body || ''));
});
const deleteBatch: DeleteMessageBatchRequest = {
QueueUrl: this.requestQueueURL,
Entries: entries
};
try {
await this.sqsClient.send(new DeleteMessageBatchCommand(deleteBatch))
} catch (err: any) {
this.logger.error("Failed to delete messages from queue.", err.message);
}
} else {
let pollDuration = new Date().getTime() - pollStartTs;
if (pollDuration < this.pollInterval) {
await sleep(this.pollInterval - pollDuration);
}
}
}
} catch (e: any) {
this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
this.logger.error(e.stack);
await this.exit(-1);
}
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
let msgBody = JSON.stringify(
{
key: scriptId,
data: [...rawResponse],
headers: headers
});
let responseQueueUrl = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(responseTopic));
if (!responseQueueUrl) {
responseQueueUrl = await this.createQueue(responseTopic);
this.queueUrls.set(responseTopic, responseQueueUrl);
}
let msgId = uuid();
let params: SendMessageRequest = {
MessageBody: msgBody,
QueueUrl: responseQueueUrl,
MessageGroupId: msgId,
MessageDeduplicationId: msgId
};
return this.sqsClient.send(new SendMessageCommand(params))
}
private async getQueues(): Promise<ListQueuesResult> {
return this.sqsClient.send(new ListQueuesCommand({}));
}
private parseQueueProperties() {
const props = this.queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
this.queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
});
}
private static topicToSqsQueueName(topic: string): string {
return topic.replace(/\./g, '_') + '.fifo';
}
private async createQueue(topic: string): Promise<string> {
let queueName = AwsSqsTemplate.topicToSqsQueueName(topic);
let queueParams: CreateQueueRequest = {
QueueName: queueName,
Attributes: this.queueAttributes
};
const result = await this.sqsClient.send(new CreateQueueCommand(queueParams));
return result.QueueUrl || '';
}
static async build(): Promise<AwsSqsTemplate> {
const queue = new AwsSqsTemplate();
await queue.init();
return queue;
}
async exit(status: number) {
this.stopped = true;
this.logger.info('Exiting with status: %d ...', status);
if (this.sqsClient) {
this.logger.info('Stopping Aws Sqs client.')
try {
this.sqsClient.destroy();
// @ts-ignore
delete this.sqsClient;
this.logger.info('Aws Sqs client stopped.')
process.exit(status);
} catch (e: any) {
this.logger.info('Aws Sqs client stop error.');
process.exit(status);
}
} else {
process.exit(status);
}
}
}

296
msa/js-executor/queue/kafkaTemplate.js
View File

@ -1,296 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const {logLevel, Kafka, CompressionTypes, Partitioners} = require('kafkajs');
const config = require('config'),
JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
logger = require('../config/logger')._logger('kafkaTemplate'),
KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator;
const replicationFactor = Number(config.get('kafka.replication_factor'));
const topicProperties = config.get('kafka.topic_properties');
const kafkaClientId = config.get('kafka.client_id');
const acks = Number(config.get('kafka.acks'));
const maxBatchSize = Number(config.get('kafka.batch_size'));
const linger = Number(config.get('kafka.linger_ms'));
const requestTimeout = Number(config.get('kafka.requestTimeout'));
const compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None;
const partitionsConsumedConcurrently = Number(config.get('kafka.partitions_consumed_concurrently'));
let kafkaClient;
let kafkaAdmin;
let consumer;
let producer;
const configEntries = [];
let batchMessages = [];
let sendLoopInstance;
function KafkaProducer() {
this.send = async (responseTopic, scriptId, rawResponse, headers) => {
logger.debug('Pending queue response, scriptId: [%s]', scriptId);
const message = {
topic: responseTopic,
messages: [{
key: scriptId,
value: rawResponse,
headers: headers.data
}]
};
await pushMessageToSendLater(message);
}
}
async function pushMessageToSendLater(message) {
batchMessages.push(message);
if (batchMessages.length >= maxBatchSize) {
await sendMessagesAsBatch(true);
}
}
function sendLoopWithLinger() {
if (sendLoopInstance) {
clearTimeout(sendLoopInstance);
// } else {
// logger.debug("Starting new send loop with linger [%s]", linger)
}
sendLoopInstance = setTimeout(sendMessagesAsBatch, linger);
}
async function sendMessagesAsBatch(isImmediately) {
if (sendLoopInstance) {
// logger.debug("sendMessagesAsBatch: Clear sendLoop scheduler. Starting new send loop with linger [%s]", linger);
clearTimeout(sendLoopInstance);
}
sendLoopInstance = null;
if (batchMessages.length > 0) {
logger.debug('sendMessagesAsBatch, length: [%s], %s', batchMessages.length, isImmediately ? 'immediately' : '');
const messagesToSend = batchMessages;
batchMessages = [];
try {
await producer.sendBatch({
topicMessages: messagesToSend,
acks: acks,
compression: compressionType
})
logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length);
} catch(err) {
logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length);
logger.error(err.stack);
batchMessages = messagesToSend.concat(batchMessages);
}
}
sendLoopWithLinger();
}
(async () => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const kafkaBootstrapServers = config.get('kafka.bootstrap.servers');
const requestTopic = config.get('request_topic');
const useConfluent = config.get('kafka.use_confluent_cloud');
logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
logger.info('Kafka Requests Topic: %s', requestTopic);
let kafkaConfig = {
brokers: kafkaBootstrapServers.split(','),
logLevel: logLevel.INFO,
logCreator: KafkaJsWinstonLogCreator
};
if (kafkaClientId) {
kafkaConfig['clientId'] = kafkaClientId;
} else {
logger.warn('KAFKA_CLIENT_ID is undefined. Consider to define the env variable KAFKA_CLIENT_ID');
}
kafkaConfig['requestTimeout'] = requestTimeout;
if (useConfluent) {
kafkaConfig['sasl'] = {
mechanism: config.get('kafka.confluent.sasl.mechanism'),
username: config.get('kafka.confluent.username'),
password: config.get('kafka.confluent.password')
};
kafkaConfig['ssl'] = true;
}
kafkaClient = new Kafka(kafkaConfig);
parseTopicProperties();
kafkaAdmin = kafkaClient.admin();
await kafkaAdmin.connect();
let partitions = 1;
for (let i = 0; i < configEntries.length; i++) {
let param = configEntries[i];
if (param.name === 'partitions') {
partitions = param.value;
configEntries.splice(i, 1);
break;
}
}
let topics = await kafkaAdmin.listTopics();
if (!topics.includes(requestTopic)) {
let createRequestTopicResult = await createTopic(requestTopic, partitions);
if (createRequestTopicResult) {
logger.info('Created new topic: %s', requestTopic);
}
}
consumer = kafkaClient.consumer({groupId: 'js-executor-group'});
producer = kafkaClient.producer({ createPartitioner: Partitioners.DefaultPartitioner });
/*
//producer event instrumentation to debug
const { CONNECT } = producer.events;
const removeListenerC = producer.on(CONNECT, e => logger.info(`producer CONNECT`));
const { DISCONNECT } = producer.events;
const removeListenerD = producer.on(DISCONNECT, e => logger.info(`producer DISCONNECT`));
const { REQUEST } = producer.events;
const removeListenerR = producer.on(REQUEST, e => logger.info(`producer REQUEST ${e.payload.broker}`));
const { REQUEST_TIMEOUT } = producer.events;
const removeListenerRT = producer.on(REQUEST_TIMEOUT, e => logger.info(`producer REQUEST_TIMEOUT ${e.payload.broker}`));
const { REQUEST_QUEUE_SIZE } = producer.events;
const removeListenerRQS = producer.on(REQUEST_QUEUE_SIZE, e => logger.info(`producer REQUEST_QUEUE_SIZE ${e.payload.broker} size ${e.queueSize}`));
*/
/*
//consumer event instrumentation to debug
const removeListeners = {}
const { FETCH_START } = consumer.events;
removeListeners[FETCH_START] = consumer.on(FETCH_START, e => logger.info(`consumer FETCH_START`));
const { FETCH } = consumer.events;
removeListeners[FETCH] = consumer.on(FETCH, e => logger.info(`consumer FETCH numberOfBatches ${e.payload.numberOfBatches} duration ${e.payload.duration}`));
const { START_BATCH_PROCESS } = consumer.events;
removeListeners[START_BATCH_PROCESS] = consumer.on(START_BATCH_PROCESS, e => logger.info(`consumer START_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`));
const { END_BATCH_PROCESS } = consumer.events;
removeListeners[END_BATCH_PROCESS] = consumer.on(END_BATCH_PROCESS, e => logger.info(`consumer END_BATCH_PROCESS topic ${e.payload.topic} batchSize ${e.payload.batchSize}`));
const { COMMIT_OFFSETS } = consumer.events;
removeListeners[COMMIT_OFFSETS] = consumer.on(COMMIT_OFFSETS, e => logger.info(`consumer COMMIT_OFFSETS topics ${e.payload.topics}`));
*/
const { CRASH } = consumer.events;
consumer.on(CRASH, e => {
logger.error(`Got consumer CRASH event, should restart: ${e.payload.restart}`);
if (!e.payload.restart) {
logger.error('Going to exit due to not retryable error!');
exit(-1);
}
});
const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer());
await consumer.connect();
await producer.connect();
sendLoopWithLinger();
await consumer.subscribe({topic: requestTopic});
logger.info('Started ThingsBoard JavaScript Executor Microservice.');
await consumer.run({
partitionsConsumedConcurrently: partitionsConsumedConcurrently,
eachMessage: async ({topic, partition, message}) => {
let headers = message.headers;
let key = message.key;
let msg = {};
msg.key = key.toString('utf8');
msg.data = message.value;
msg.headers = {data: headers};
messageProcessor.onJsInvokeMessage(msg);
},
});
} catch (e) {
logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
logger.error(e.stack);
exit(-1);
}
})();
function createTopic(topic, partitions) {
return kafkaAdmin.createTopics({
topics: [{
topic: topic,
numPartitions: partitions,
replicationFactor: replicationFactor,
configEntries: configEntries
}]
});
}
function parseTopicProperties() {
const props = topicProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
configEntries.push({name: p.substring(0, delimiterPosition), value: p.substring(delimiterPosition + 1)});
});
}
process.on('exit', () => {
exit(0);
});
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
if (kafkaAdmin) {
logger.info('Stopping Kafka Admin...');
await kafkaAdmin.disconnect();
logger.info('Kafka Admin stopped.');
}
if (consumer) {
logger.info('Stopping Kafka Consumer...');
let _consumer = consumer;
consumer = null;
try {
await _consumer.disconnect();
logger.info('Kafka Consumer stopped.');
await disconnectProducer();
process.exit(status);
} catch (e) {
logger.info('Kafka Consumer stop error.');
await disconnectProducer();
process.exit(status);
}
} else {
process.exit(status);
}
}
async function disconnectProducer() {
if (producer) {
logger.info('Stopping Kafka Producer...');
var _producer = producer;
producer = null;
try {
logger.info('Stopping loop...');
clearTimeout(sendLoopInstance);
await sendMessagesAsBatch();
await _producer.disconnect();
logger.info('Kafka Producer stopped.');
} catch (e) {
logger.info('Kafka Producer stop error.');
}
}
}

291
msa/js-executor/queue/kafkaTemplate.ts
View File

@ -0,0 +1,291 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import { _logger, KafkaJsWinstonLogCreator } from '../config/logger';
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
import { IQueue } from './queue.models';
import {
Admin,
CompressionTypes,
Consumer,
Kafka,
KafkaConfig,
logLevel,
Partitioners,
Producer,
TopicMessages
} from 'kafkajs';
export class KafkaTemplate implements IQueue {
private logger = _logger(`kafkaTemplate`);
private replicationFactor = Number(config.get('kafka.replication_factor'));
private topicProperties: string = config.get('kafka.topic_properties');
private kafkaClientId: string = config.get('kafka.client_id');
private acks = Number(config.get('kafka.acks'));
private maxBatchSize = Number(config.get('kafka.batch_size'));
private linger = Number(config.get('kafka.linger_ms'));
private requestTimeout = Number(config.get('kafka.requestTimeout'));
private compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None;
private partitionsConsumedConcurrently = Number(config.get('kafka.partitions_consumed_concurrently'));
private kafkaClient: Kafka;
private kafkaAdmin: Admin;
private consumer: Consumer;
private producer: Producer;
private configEntries: any[] = [];
private batchMessages: TopicMessages[] = [];
private sendLoopInstance: NodeJS.Timeout;
constructor() {
}
async init(): Promise<void> {
try {
this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const kafkaBootstrapServers: string = config.get('kafka.bootstrap.servers');
const requestTopic: string = config.get('request_topic');
const useConfluent = config.get('kafka.use_confluent_cloud');
this.logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
this.logger.info('Kafka Requests Topic: %s', requestTopic);
let kafkaConfig: KafkaConfig = {
brokers: kafkaBootstrapServers.split(','),
logLevel: logLevel.INFO,
logCreator: KafkaJsWinstonLogCreator
};
if (this.kafkaClientId) {
kafkaConfig['clientId'] = this.kafkaClientId;
} else {
this.logger.warn('KAFKA_CLIENT_ID is undefined. Consider to define the env variable KAFKA_CLIENT_ID');
}
kafkaConfig['requestTimeout'] = this.requestTimeout;
if (useConfluent) {
kafkaConfig['sasl'] = {
mechanism: config.get('kafka.confluent.sasl.mechanism') as any,
username: config.get('kafka.confluent.username'),
password: config.get('kafka.confluent.password')
};
kafkaConfig['ssl'] = true;
}
this.parseTopicProperties();
this.kafkaClient = new Kafka(kafkaConfig);
this.kafkaAdmin = this.kafkaClient.admin();
await this.kafkaAdmin.connect();
let partitions = 1;
for (let i = 0; i < this.configEntries.length; i++) {
let param = this.configEntries[i];
if (param.name === 'partitions') {
partitions = param.value;
this.configEntries.splice(i, 1);
break;
}
}
let topics = await this.kafkaAdmin.listTopics();
if (!topics.includes(requestTopic)) {
let createRequestTopicResult = await this.createTopic(requestTopic, partitions);
if (createRequestTopicResult) {
this.logger.info('Created new topic: %s', requestTopic);
}
}
this.consumer = this.kafkaClient.consumer({groupId: 'js-executor-group'});
this.producer = this.kafkaClient.producer({createPartitioner: Partitioners.DefaultPartitioner});
const {CRASH} = this.consumer.events;
this.consumer.on(CRASH, e => {
this.logger.error(`Got consumer CRASH event, should restart: ${e.payload.restart}`);
if (!e.payload.restart) {
this.logger.error('Going to exit due to not retryable error!');
this.exit(-1);
}
});
const messageProcessor = new JsInvokeMessageProcessor(this);
await this.consumer.connect();
await this.producer.connect();
this.sendLoopWithLinger();
await this.consumer.subscribe({topic: requestTopic});
this.logger.info('Started ThingsBoard JavaScript Executor Microservice.');
await this.consumer.run({
partitionsConsumedConcurrently: this.partitionsConsumedConcurrently,
eachMessage: async ({topic, partition, message}) => {
let headers = message.headers;
let key = message.key || new Buffer([]);
let msg = {
key: key.toString('utf8'),
data: message.value,
headers: {
data: headers
}
};
messageProcessor.onJsInvokeMessage(msg);
},
});
} catch (e: any) {
this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
this.logger.error(e.stack);
await this.exit(-1);
}
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
this.logger.debug('Pending queue response, scriptId: [%s]', scriptId);
const message = {
topic: responseTopic,
messages: [{
key: scriptId,
value: rawResponse,
headers: headers.data
}]
};
await this.pushMessageToSendLater(message);
}
private async pushMessageToSendLater(message: TopicMessages) {
this.batchMessages.push(message);
if (this.batchMessages.length >= this.maxBatchSize) {
await this.sendMessagesAsBatch(true);
}
}
private async sendMessagesAsBatch(isImmediately = false): Promise<void> {
if (this.sendLoopInstance) {
clearTimeout(this.sendLoopInstance);
}
if (this.batchMessages.length > 0) {
this.logger.debug('sendMessagesAsBatch, length: [%s], %s', this.batchMessages.length, isImmediately ? 'immediately' : '');
const messagesToSend = this.batchMessages;
this.batchMessages = [];
try {
await this.producer.sendBatch({
topicMessages: messagesToSend,
acks: this.acks,
compression: this.compressionType
})
this.logger.debug('Response batch sent to kafka, length: [%s]', messagesToSend.length);
} catch (err: any) {
this.logger.error('Failed batch send to kafka, length: [%s], pending to reprocess msgs', messagesToSend.length);
this.logger.error(err.stack);
this.batchMessages = messagesToSend.concat(this.batchMessages);
}
}
this.sendLoopWithLinger();
}
private parseTopicProperties() {
const props = this.topicProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
this.configEntries.push({
name: p.substring(0, delimiterPosition),
value: p.substring(delimiterPosition + 1)
});
});
}
private createTopic(topic: string, partitions: number): Promise<boolean> {
return this.kafkaAdmin.createTopics({
topics: [{
topic: topic,
numPartitions: partitions,
replicationFactor: this.replicationFactor,
configEntries: this.configEntries
}]
});
}
private sendLoopWithLinger() {
if (this.sendLoopInstance) {
clearTimeout(this.sendLoopInstance);
// } else {
// this.logger.debug("Starting new send loop with linger [%s]", this.linger)
}
this.sendLoopInstance = setTimeout(async () => {
await this.sendMessagesAsBatch()
}, this.linger);
}
static async build(): Promise<KafkaTemplate> {
const queue = new KafkaTemplate();
await queue.init();
return queue;
}
async exit(status: number): Promise<void> {
this.logger.info('Exiting with status: %d ...', status);
if (this.kafkaAdmin) {
this.logger.info('Stopping Kafka Admin...');
await this.kafkaAdmin.disconnect();
// @ts-ignore
delete this.kafkaAdmin;
this.logger.info('Kafka Admin stopped.');
}
if (this.consumer) {
this.logger.info('Stopping Kafka Consumer...');
try {
await this.consumer.disconnect();
// @ts-ignore
delete this.consumer;
this.logger.info('Kafka Consumer stopped.');
await this.disconnectProducer();
process.exit(status);
} catch (e: any) {
this.logger.info('Kafka Consumer stop error.');
await this.disconnectProducer();
process.exit(status);
}
} else {
process.exit(status);
}
}
private async disconnectProducer(): Promise<void> {
if (this.producer) {
this.logger.info('Stopping Kafka Producer...');
try {
this.logger.info('Stopping loop...');
clearTimeout(this.sendLoopInstance);
await this.sendMessagesAsBatch();
await this.producer.disconnect();
// @ts-ignore
delete this.producer;
this.logger.info('Kafka Producer stopped.');
} catch (e) {
this.logger.info('Kafka Producer stop error.');
}
}
}
}

162
msa/js-executor/queue/pubSubTemplate.js
View File

@ -1,162 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const config = require('config'),
JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
logger = require('../config/logger')._logger('pubSubTemplate');
const {PubSub} = require('@google-cloud/pubsub');
const projectId = config.get('pubsub.project_id');
const credentials = JSON.parse(config.get('pubsub.service_account'));
const requestTopic = config.get('request_topic');
const queueProperties = config.get('pubsub.queue_properties');
let pubSubClient;
const topics = [];
const subscriptions = [];
const queueProps = [];
function PubSubProducer() {
this.send = async (responseTopic, scriptId, rawResponse, headers) => {
if (!(subscriptions.includes(responseTopic) && topics.includes(requestTopic))) {
await createTopic(requestTopic);
}
let data = JSON.stringify(
{
key: scriptId,
data: [...rawResponse],
headers: headers
});
let dataBuffer = Buffer.from(data);
return pubSubClient.topic(responseTopic).publish(dataBuffer);
}
}
(async () => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
pubSubClient = new PubSub({projectId: projectId, credentials: credentials});
parseQueueProperties();
const topicList = await pubSubClient.getTopics();
if (topicList) {
topicList[0].forEach(topic => {
topics.push(getName(topic.name));
});
}
const subscriptionList = await pubSubClient.getSubscriptions();
if (subscriptionList) {
topicList[0].forEach(sub => {
subscriptions.push(getName(sub.name));
});
}
if (!(subscriptions.includes(requestTopic) && topics.includes(requestTopic))) {
await createTopic(requestTopic);
}
const subscription = pubSubClient.subscription(requestTopic);
const messageProcessor = new JsInvokeMessageProcessor(new PubSubProducer());
const messageHandler = message => {
messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8')));
message.ack();
};
subscription.on('message', messageHandler);
} catch (e) {
logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
logger.error(e.stack);
exit(-1);
}
})();
async function createTopic(topic) {
if (!topics.includes(topic)) {
try {
await pubSubClient.createTopic(topic);
logger.info('Created new Pub/Sub topic: %s', topic);
} catch (e) {
logger.info('Pub/Sub topic already exists');
}
topics.push(topic);
}
await createSubscription(topic)
}
async function createSubscription(topic) {
if (!subscriptions.includes(topic)) {
try {
await pubSubClient.createSubscription(topic, topic, {
topic: topic,
subscription: topic,
ackDeadlineSeconds: queueProps['ackDeadlineInSec'],
messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']}
});
logger.info('Created new Pub/Sub subscription: %s', topic);
} catch (e) {
logger.info('Pub/Sub subscription already exists.');
}
subscriptions.push(topic);
}
}
function parseQueueProperties() {
const props = queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
});
}
function getName(fullName) {
const delimiterPosition = fullName.lastIndexOf('/');
return fullName.substring(delimiterPosition + 1);
}
process.on('exit', () => {
exit(0);
});
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
if (pubSubClient) {
logger.info('Stopping Pub/Sub client.')
try {
await pubSubClient.close();
logger.info('Pub/Sub client stopped.')
process.exit(status);
} catch (e) {
logger.info('Pub/Sub client stop error.');
process.exit(status);
}
} else {
process.exit(status);
}
}

175
msa/js-executor/queue/pubSubTemplate.ts
View File

@ -0,0 +1,175 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import { _logger } from '../config/logger';
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
import { PubSub } from '@google-cloud/pubsub';
import { IQueue } from './queue.models';
import { Message } from '@google-cloud/pubsub/build/src/subscriber';
export class PubSubTemplate implements IQueue {
private logger = _logger(`pubSubTemplate`);
private projectId: string = config.get('pubsub.project_id');
private credentials = JSON.parse(config.get('pubsub.service_account'));
private requestTopic: string = config.get('request_topic');
private queueProperties: string = config.get('pubsub.queue_properties');
private pubSubClient: PubSub;
private queueProps: { [n: string]: string } = {};
private topics: string[] = [];
private subscriptions: string[] = [];
constructor() {
}
async init() {
try {
this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
this.pubSubClient = new PubSub({
projectId: this.projectId,
credentials: this.credentials
});
this.parseQueueProperties();
const topicList = await this.pubSubClient.getTopics();
if (topicList) {
topicList[0].forEach(topic => {
this.topics.push(PubSubTemplate.getName(topic.name));
});
}
const subscriptionList = await this.pubSubClient.getSubscriptions();
if (subscriptionList) {
topicList[0].forEach(sub => {
this.subscriptions.push(PubSubTemplate.getName(sub.name));
});
}
if (!(this.subscriptions.includes(this.requestTopic) && this.topics.includes(this.requestTopic))) {
await this.createTopic(this.requestTopic);
await this.createSubscription(this.requestTopic);
}
const subscription = this.pubSubClient.subscription(this.requestTopic);
const messageProcessor = new JsInvokeMessageProcessor(this);
const messageHandler = (message: Message) => {
messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8')));
message.ack();
};
subscription.on('message', messageHandler);
} catch (e: any) {
this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
this.logger.error(e.stack);
await this.exit(-1);
}
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
if (!(this.subscriptions.includes(responseTopic) && this.topics.includes(this.requestTopic))) {
await this.createTopic(this.requestTopic);
await this.createSubscription(this.requestTopic);
}
let data = JSON.stringify(
{
key: scriptId,
data: [...rawResponse],
headers: headers
});
let dataBuffer = Buffer.from(data);
return this.pubSubClient.topic(responseTopic).publishMessage({data: dataBuffer});
}
private parseQueueProperties() {
const props = this.queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
this.queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
});
}
private static getName(fullName: string): string {
const delimiterPosition = fullName.lastIndexOf('/');
return fullName.substring(delimiterPosition + 1);
}
private async createTopic(topic: string) {
if (!this.topics.includes(topic)) {
try {
await this.pubSubClient.createTopic(topic);
this.logger.info('Created new Pub/Sub topic: %s', topic);
} catch (e) {
this.logger.info('Pub/Sub topic already exists');
}
this.topics.push(topic);
}
}
private async createSubscription(topic: string) {
if (!this.subscriptions.includes(topic)) {
try {
await this.pubSubClient.createSubscription(topic, topic, {
topic: topic,
name: topic,
ackDeadlineSeconds: Number(this.queueProps['ackDeadlineInSec']),
messageRetentionDuration: {
seconds: this.queueProps['messageRetentionInSec']
}
});
this.logger.info('Created new Pub/Sub subscription: %s', topic);
} catch (e) {
this.logger.info('Pub/Sub subscription already exists.');
}
this.subscriptions.push(topic);
}
}
static async build(): Promise<PubSubTemplate> {
const queue = new PubSubTemplate();
await queue.init();
return queue;
}
async exit(status: number): Promise<void> {
this.logger.info('Exiting with status: %d ...', status);
if (this.pubSubClient) {
this.logger.info('Stopping Pub/Sub client.')
try {
await this.pubSubClient.close();
// @ts-ignore
delete this.pubSubClient;
this.logger.info('Pub/Sub client stopped.')
process.exit(status);
} catch (e) {
this.logger.info('Pub/Sub client stop error.');
process.exit(status);
}
} else {
process.exit(status);
}
}
}

21
msa/js-executor/queue/queue.models.ts
View File

@ -0,0 +1,21 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
export interface IQueue {
init(): Promise<void>;
send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any>;
exit(status: number): Promise<void>;
}

181
msa/js-executor/queue/rabbitmqTemplate.js
View File

@ -1,181 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const config = require('config'),
JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
logger = require('../config/logger')._logger('rabbitmqTemplate');
const requestTopic = config.get('request_topic');
const host = config.get('rabbitmq.host');
const port = config.get('rabbitmq.port');
const vhost = config.get('rabbitmq.virtual_host');
const username = config.get('rabbitmq.username');
const password = config.get('rabbitmq.password');
const queueProperties = config.get('rabbitmq.queue_properties');
const pollInterval = config.get('js.response_poll_interval');
const amqp = require('amqplib/callback_api');
let queueOptions = {durable: false, exclusive: false, autoDelete: false};
let connection;
let channel;
let stopped = false;
let queues = [];
function RabbitMqProducer() {
this.send = async (responseTopic, scriptId, rawResponse, headers) => {
if (!queues.includes(responseTopic)) {
await createQueue(responseTopic);
queues.push(responseTopic);
}
let data = JSON.stringify(
{
key: scriptId,
data: [...rawResponse],
headers: headers
});
let dataBuffer = Buffer.from(data);
channel.sendToQueue(responseTopic, dataBuffer);
return new Promise((resolve, reject) => {
channel.waitForConfirms((err) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
}
(async () => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const url = `amqp://${username}:${password}@${host}:${port}${vhost}`;
connection = await new Promise((resolve, reject) => {
amqp.connect(url, function (err, connection) {
if (err) {
reject(err);
} else {
resolve(connection);
}
});
});
channel = await new Promise((resolve, reject) => {
connection.createConfirmChannel(function (err, channel) {
if (err) {
reject(err);
} else {
resolve(channel);
}
});
});
parseQueueProperties();
await createQueue(requestTopic);
const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer());
while (!stopped) {
let pollStartTs = new Date().getTime();
let message = await new Promise((resolve, reject) => {
channel.get(requestTopic, {}, function (err, msg) {
if (err) {
reject(err);
} else {
resolve(msg);
}
});
});
if (message) {
messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8')));
channel.ack(message);
} else {
let pollDuration = new Date().getTime() - pollStartTs;
if (pollDuration < pollInterval) {
await sleep(pollInterval - pollDuration);
}
}
}
} catch (e) {
logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
logger.error(e.stack);
exit(-1);
}
})();
function parseQueueProperties() {
let args = {};
const props = queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
args[p.substring(0, delimiterPosition)] = +p.substring(delimiterPosition + 1);
});
queueOptions['arguments'] = args;
}
async function createQueue(topic) {
return new Promise((resolve, reject) => {
channel.assertQueue(topic, queueOptions, function (err) {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
process.on('exit', () => {
exit(0);
});
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
if (channel) {
logger.info('Stopping RabbitMq chanel.')
await channel.close();
logger.info('RabbitMq chanel stopped');
}
if (connection) {
logger.info('Stopping RabbitMq connection.')
try {
await connection.close();
logger.info('RabbitMq client connection.')
process.exit(status);
} catch (e) {
logger.info('RabbitMq connection stop error.');
process.exit(status);
}
} else {
process.exit(status);
}
}

151
msa/js-executor/queue/rabbitmqTemplate.ts
View File

@ -0,0 +1,151 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import { _logger } from '../config/logger';
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
import { IQueue } from './queue.models';
import amqp, { ConfirmChannel, Connection } from 'amqplib';
import { Options, Replies } from 'amqplib/properties';
import { sleep } from '../api/utils';
export class RabbitMqTemplate implements IQueue {
private logger = _logger(`rabbitmqTemplate`);
private requestTopic: string = config.get('request_topic');
private host = config.get('rabbitmq.host');
private port = config.get('rabbitmq.port');
private vhost = config.get('rabbitmq.virtual_host');
private username = config.get('rabbitmq.username');
private password = config.get('rabbitmq.password');
private queueProperties: string = config.get('rabbitmq.queue_properties');
private pollInterval = Number(config.get('js.response_poll_interval'));
private queueOptions: Options.AssertQueue = {
durable: false,
exclusive: false,
autoDelete: false
};
private connection: Connection;
private channel: ConfirmChannel;
private stopped = false;
private topics: string[] = [];
constructor() {
}
async init(): Promise<void> {
try {
this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const url = `amqp://${this.username}:${this.password}@${this.host}:${this.port}${this.vhost}`;
this.connection = await amqp.connect(url);
this.channel = await this.connection.createConfirmChannel();
this.parseQueueProperties();
await this.createQueue(this.requestTopic);
const messageProcessor = new JsInvokeMessageProcessor(this);
while (!this.stopped) {
let pollStartTs = new Date().getTime();
let message = await this.channel.get(this.requestTopic);
if (message) {
messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8')));
this.channel.ack(message);
} else {
let pollDuration = new Date().getTime() - pollStartTs;
if (pollDuration < this.pollInterval) {
await sleep(this.pollInterval - pollDuration);
}
}
}
} catch (e: any) {
this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
this.logger.error(e.stack);
await this.exit(-1);
}
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
if (!this.topics.includes(responseTopic)) {
await this.createQueue(responseTopic);
this.topics.push(responseTopic);
}
let data = JSON.stringify(
{
key: scriptId,
data: [...rawResponse],
headers: headers
});
let dataBuffer = Buffer.from(data);
this.channel.sendToQueue(responseTopic, dataBuffer);
return this.channel.waitForConfirms()
}
private parseQueueProperties() {
let args: { [n: string]: number } = {};
const props = this.queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
args[p.substring(0, delimiterPosition)] = Number(p.substring(delimiterPosition + 1));
});
this.queueOptions['arguments'] = args;
}
private async createQueue(topic: string): Promise<Replies.AssertQueue> {
return this.channel.assertQueue(topic, this.queueOptions);
}
static async build(): Promise<RabbitMqTemplate> {
const queue = new RabbitMqTemplate();
await queue.init();
return queue;
}
async exit(status: number) {
this.logger.info('Exiting with status: %d ...', status);
if (this.channel) {
this.logger.info('Stopping RabbitMq chanel.')
await this.channel.close();
// @ts-ignore
delete this.channel;
this.logger.info('RabbitMq chanel stopped');
}
if (this.connection) {
this.logger.info('Stopping RabbitMq connection.')
try {
await this.connection.close();
// @ts-ignore
delete this.connection;
this.logger.info('RabbitMq client connection.')
process.exit(status);
} catch (e) {
this.logger.info('RabbitMq connection stop error.');
process.exit(status);
}
} else {
process.exit(status);
}
}
}

177
msa/js-executor/queue/serviceBusTemplate.js
View File

@ -1,177 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const config = require('config'),
JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
logger = require('../config/logger')._logger('serviceBusTemplate');
const {ServiceBusClient, ServiceBusAdministrationClient} = require("@azure/service-bus");
const requestTopic = config.get('request_topic');
const namespaceName = config.get('service_bus.namespace_name');
const sasKeyName = config.get('service_bus.sas_key_name');
const sasKey = config.get('service_bus.sas_key');
const queueProperties = config.get('service_bus.queue_properties');
let sbClient;
let receiver;
let serviceBusService;
let queueOptions = {};
const queues = [];
const senderMap = new Map();
function ServiceBusProducer() {
this.send = async (responseTopic, scriptId, rawResponse, headers) => {
if (!queues.includes(requestTopic)) {
await createQueueIfNotExist(requestTopic);
queues.push(requestTopic);
}
let customSender = senderMap.get(responseTopic);
if (!customSender) {
customSender = new CustomSender(responseTopic);
senderMap.set(responseTopic, customSender);
}
let data = {
key: scriptId,
data: [...rawResponse],
headers: headers
};
return customSender.send({body: data});
}
}
function CustomSender(topic) {
this.sender = sbClient.createSender(topic);
this.send = async (message) => {
return this.sender.sendMessages(message);
}
}
(async () => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const connectionString = `Endpoint=sb://${namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${sasKeyName};SharedAccessKey=${sasKey}`;
sbClient = new ServiceBusClient(connectionString)
serviceBusService = new ServiceBusAdministrationClient(connectionString);
parseQueueProperties();
await new Promise((resolve, reject) => {
serviceBusService.listQueues((err, data) => {
if (err) {
reject(err);
} else {
for (const queue of data) {
queues.push(queue.name);
}
resolve();
}
});
});
if (!queues.includes(requestTopic)) {
await createQueueIfNotExist(requestTopic);
queues.push(requestTopic);
}
receiver = sbClient.createReceiver(requestTopic, {receiveMode: 'peekLock'});
const messageProcessor = new JsInvokeMessageProcessor(new ServiceBusProducer());
const messageHandler = async (message) => {
if (message) {
messageProcessor.onJsInvokeMessage(message.body);
await message.complete();
}
};
const errorHandler = (error) => {
logger.error('Failed to receive message from queue.', error);
};
receiver.subscribe({processMessage: messageHandler, processError: errorHandler})
} catch (e) {
logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
logger.error(e.stack);
await exit(-1);
}
})();
async function createQueueIfNotExist(topic) {
return new Promise((resolve, reject) => {
serviceBusService.createQueue(topic, queueOptions, (err) => {
if (err && err.code !== "MessageEntityAlreadyExistsError") {
reject(err);
} else {
resolve();
}
});
});
}
function parseQueueProperties() {
let properties = {};
const props = queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
});
queueOptions = {
requiresDuplicateDetection: false,
maxSizeInMegabytes: properties['maxSizeInMb'],
defaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`,
lockDuration: `PT${properties['lockDurationInSec']}S`
};
}
process.on('exit', () => {
exit(0);
});
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
logger.info('Stopping Azure Service Bus resources...')
if (receiver) {
try {
await receiver.close();
} catch (e) {
}
}
senderMap.forEach((k, v) => {
try {
v.sender.close();
} catch (e) {
}
});
if (sbClient) {
try {
sbClient.close();
} catch (e) {
}
}
logger.info('Azure Service Bus resources stopped.')
process.exit(status);
}

175
msa/js-executor/queue/serviceBusTemplate.ts
View File

@ -0,0 +1,175 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import { _logger } from '../config/logger';
import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor'
import { IQueue } from './queue.models';
import {
CreateQueueOptions,
ProcessErrorArgs,
ServiceBusAdministrationClient,
ServiceBusClient,
ServiceBusReceivedMessage,
ServiceBusReceiver,
ServiceBusSender
} from '@azure/service-bus';
export class ServiceBusTemplate implements IQueue {
private logger = _logger(`serviceBusTemplate`);
private requestTopic: string = config.get('request_topic');
private namespaceName = config.get('service_bus.namespace_name');
private sasKeyName = config.get('service_bus.sas_key_name');
private sasKey = config.get('service_bus.sas_key');
private queueProperties: string = config.get('service_bus.queue_properties');
private sbClient: ServiceBusClient;
private serviceBusService: ServiceBusAdministrationClient;
private queueOptions: CreateQueueOptions = {};
private queues: string[] = [];
private receiver: ServiceBusReceiver;
private senderMap = new Map<string, ServiceBusSender>();
constructor() {
}
async init() {
try {
this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const connectionString = `Endpoint=sb://${this.namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${this.sasKeyName};SharedAccessKey=${this.sasKey}`;
this.sbClient = new ServiceBusClient(connectionString)
this.serviceBusService = new ServiceBusAdministrationClient(connectionString);
this.parseQueueProperties();
const listQueues = await this.serviceBusService.listQueues();
for await (const queue of listQueues) {
this.queues.push(queue.name);
}
if (!this.queues.includes(this.requestTopic)) {
await this.createQueueIfNotExist(this.requestTopic);
this.queues.push(this.requestTopic);
}
this.receiver = this.sbClient.createReceiver(this.requestTopic, {receiveMode: 'peekLock'});
const messageProcessor = new JsInvokeMessageProcessor(this);
const messageHandler = async (message: ServiceBusReceivedMessage) => {
if (message) {
messageProcessor.onJsInvokeMessage(message.body);
await this.receiver.completeMessage(message);
}
};
const errorHandler = async (error: ProcessErrorArgs) => {
this.logger.error('Failed to receive message from queue.', error);
};
this.receiver.subscribe({processMessage: messageHandler, processError: errorHandler})
} catch (e: any) {
this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
this.logger.error(e.stack);
await this.exit(-1);
}
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
if (!this.queues.includes(this.requestTopic)) {
await this.createQueueIfNotExist(this.requestTopic);
this.queues.push(this.requestTopic);
}
let customSender = this.senderMap.get(responseTopic);
if (!customSender) {
customSender = this.sbClient.createSender(responseTopic);
this.senderMap.set(responseTopic, customSender);
}
let data = {
key: scriptId,
data: [...rawResponse],
headers: headers
};
return customSender.sendMessages({body: data});
}
private parseQueueProperties() {
let properties: { [n: string]: string } = {};
const props = this.queueProperties.split(';');
props.forEach(p => {
const delimiterPosition = p.indexOf(':');
properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
});
this.queueOptions = {
requiresDuplicateDetection: false,
maxSizeInMegabytes: Number(properties['maxSizeInMb']),
defaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`,
lockDuration: `PT${properties['lockDurationInSec']}S`
};
}
private async createQueueIfNotExist(topic: string) {
try {
await this.serviceBusService.createQueue(topic, this.queueOptions)
} catch (err: any) {
if (err && err.code !== "MessageEntityAlreadyExistsError") {
throw new Error(err);
}
}
}
static async build(): Promise<ServiceBusTemplate> {
const queue = new ServiceBusTemplate();
await queue.init();
return queue;
}
async exit(status: number) {
this.logger.info('Exiting with status: %d ...', status);
this.logger.info('Stopping Azure Service Bus resources...')
if (this.receiver) {
try {
await this.receiver.close();
// @ts-ignore
delete this.receiver;
} catch (e) {
}
}
this.senderMap.forEach(k => {
try {
k.close();
} catch (e) {
}
});
this.senderMap.clear();
if (this.sbClient) {
try {
await this.sbClient.close();
// @ts-ignore
delete this.sbClient;
} catch (e) {
}
}
this.logger.info('Azure Service Bus resources stopped.')
process.exit(status);
}
}

55
msa/js-executor/server.js
View File

@ -1,55 +0,0 @@
/*
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const config = require('config'), logger = require('./config/logger')._logger('main');
logger.info('===CONFIG BEGIN===');
logger.info(JSON.stringify(config, null, 4));
logger.info('===CONFIG END===');
const serviceType = config.get('queue_type');
switch (serviceType) {
case 'kafka':
logger.info('Starting kafka template.');
require('./queue/kafkaTemplate');
logger.info('kafka template started.');
break;
case 'pubsub':
logger.info('Starting Pub/Sub template.')
require('./queue/pubSubTemplate');
logger.info('Pub/Sub template started.')
break;
case 'aws-sqs':
logger.info('Starting Aws Sqs template.')
require('./queue/awsSqsTemplate');
logger.info('Aws Sqs template started.')
break;
case 'rabbitmq':
logger.info('Starting RabbitMq template.')
require('./queue/rabbitmqTemplate');
logger.info('RabbitMq template started.')
break;
case 'service-bus':
logger.info('Starting Azure Service Bus template.')
require('./queue/serviceBusTemplate');
logger.info('Azure Service Bus template started.')
break;
default:
logger.error('Unknown service type: ', serviceType);
process.exit(-1);
}
require('./api/httpServer');

85
msa/js-executor/server.ts
View File

@ -0,0 +1,85 @@
///
/// Copyright © 2016-2022 The Thingsboard Authors
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
import config from 'config';
import { _logger } from './config/logger';
import { HttpServer } from './api/httpServer';
import { IQueue } from './queue/queue.models';
import { KafkaTemplate } from './queue/kafkaTemplate';
import { PubSubTemplate } from './queue/pubSubTemplate';
import { AwsSqsTemplate } from './queue/awsSqsTemplate';
import { RabbitMqTemplate } from './queue/rabbitmqTemplate';
import { ServiceBusTemplate } from './queue/serviceBusTemplate';
const logger = _logger('main');
logger.info('===CONFIG BEGIN===');
logger.info(JSON.stringify(config, null, 4));
logger.info('===CONFIG END===');
const serviceType = config.get('queue_type');
const httpPort = Number(config.get('http_port'));
let queues: IQueue;
let httpServer: HttpServer;
(async () => {
switch (serviceType) {
case 'kafka':
logger.info('Starting kafka template.');
queues = await KafkaTemplate.build();
logger.info('kafka template started.');
break;
case 'pubsub':
logger.info('Starting Pub/Sub template.')
queues = await PubSubTemplate.build();
logger.info('Pub/Sub template started.')
break;
case 'aws-sqs':
logger.info('Starting Aws Sqs template.')
queues = await AwsSqsTemplate.build();
logger.info('Aws Sqs template started.')
break;
case 'rabbitmq':
logger.info('Starting RabbitMq template.')
queues = await RabbitMqTemplate.build();
logger.info('RabbitMq template started.')
break;
case 'service-bus':
logger.info('Starting Azure Service Bus template.')
queues = await ServiceBusTemplate.build();
logger.info('Azure Service Bus template started.')
break;
default:
logger.error('Unknown service type: ', serviceType);
process.exit(-1);
}
httpServer = new HttpServer(httpPort);
})();
process.on('SIGTERM', () => {
process.exit();
});
process.on('exit', async () => {
if (httpServer) {
httpServer.stop();
}
if (queues) {
queues.exit(0);
}
});

13
msa/js-executor/tsconfig.json
View File

@ -0,0 +1,13 @@
{
"compilerOptions": {
"outDir": "target/src",
"target": "es2016",
"module": "commonjs",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": true,
"skipLibCheck": true,
"strictPropertyInitialization": false
},
"exclude": ["node_modules", "target"]
}

947
msa/js-executor/yarn.lock
View File

File diff suppressed because it is too large
Write Preview
Loading…
Cancel
Save