Improve JS Executor

application/src/main/resources/thingsboard.yml

@@ -203,7 +203,7 @@ sql:
     attributes:
       batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}"
       batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
-      stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:1000}"
+      stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}"
     # Specify whether to remove null characters from strValue of attributes and timeseries before insert
     remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
 

msa/js-executor/api/jsExecutor.js

@@ -17,11 +17,28 @@
 
 const vm = require('vm');
 
-function JsExecutor() {
+function JsExecutor(useSandbox) {
+    this.useSandbox = useSandbox;
 }
 
 JsExecutor.prototype.compileScript = function(code) {
-    return new Promise(function(resolve, reject) {
+    if (this.useSandbox) {
+        return createScript(code);
+    } else {
+        return createFunction(code);
+    }
+}
+
+JsExecutor.prototype.executeScript = function(script, args, timeout) {
+    if (this.useSandbox) {
+        return invokeScript(script, args, timeout);
+    } else {
+        return invokeFunction(script, args);
+    }
+}
+
+function createScript(code) {
+    return new Promise((resolve, reject) => {
         try {
             code = "("+code+")(...args)";
             var script = new vm.Script(code);
@@ -32,8 +49,8 @@ JsExecutor.prototype.compileScript = function(code) {
     });
 }
 
-JsExecutor.prototype.executeScript = function(script, args, timeout) {
-    return new Promise(function(resolve, reject) {
+function invokeScript(script, args, timeout) {
+    return new Promise((resolve, reject) => {
         try {
             var sandbox = Object.create(null);
             sandbox.args = args;
@@ -45,4 +62,29 @@ JsExecutor.prototype.executeScript = function(script, args, timeout) {
     });
 }
 
+
+function createFunction(code) {
+    return new Promise((resolve, reject) => {
+        try {
+            code = "return ("+code+")(...args)";
+            const parsingContext = vm.createContext({});
+            const func = vm.compileFunction(code, ['args'], {parsingContext: parsingContext});
+            resolve(func);
+        } catch (err) {
+            reject(err);
+        }
+    });
+}
+
+function invokeFunction(func, args) {
+    return new Promise((resolve, reject) => {
+        try {
+            var result = func(args);
+            resolve(result);
+        } catch (err) {
+            reject(err);
+        }
+    });
+}
+
 module.exports = JsExecutor;

msa/js-executor/api/jsInvokeMessageProcessor.js

@@ -15,18 +15,22 @@
  */
 'use strict';
 
+const COMPILATION_ERROR = 0;
+const RUNTIME_ERROR = 1;
+const TIMEOUT_ERROR = 2;
+const UNRECOGNIZED = -1;
+
 const config = require('config'),
-      logger = require('../config/logger')('JsInvokeMessageProcessor'),
+      logger = require('../config/logger')._logger('JsInvokeMessageProcessor'),
       Utils = require('./utils'),
-      js = require('./jsinvoke.proto').js,
-      KeyedMessage = require('kafka-node').KeyedMessage,
       JsExecutor = require('./jsExecutor');
 
 const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency'));
+const useSandbox = config.get('script.use_sandbox') === 'true';
 
 function JsInvokeMessageProcessor(producer) {
     this.producer = producer;
-    this.executor = new JsExecutor();
+    this.executor = new JsExecutor(useSandbox);
     this.scriptMap = {};
     this.executedScriptsCounter = 0;
 }
@@ -34,18 +38,22 @@ function JsInvokeMessageProcessor(producer) {
 JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) {
 
     var requestId;
+    var responseTopic;
     try {
-        var request = js.RemoteJsRequest.decode(message.value);
-        requestId = getRequestId(request);
+        var request = JSON.parse(message.value.toString('utf8'));
+        var buf = message.headers['requestId'];
+        requestId = Utils.UUIDFromBuffer(buf);
+        buf = message.headers['responseTopic'];
+        responseTopic = buf.toString('utf8');
 
-        logger.debug('[%s] Received request, responseTopic: [%s]', requestId, request.responseTopic);
+        logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic);
 
         if (request.compileRequest) {
-            this.processCompileRequest(requestId, request.responseTopic, request.compileRequest);
+            this.processCompileRequest(requestId, responseTopic, request.compileRequest);
         } else if (request.invokeRequest) {
-            this.processInvokeRequest(requestId, request.responseTopic, request.invokeRequest);
+            this.processInvokeRequest(requestId, responseTopic, request.invokeRequest);
         } else if (request.releaseRequest) {
-            this.processReleaseRequest(requestId, request.responseTopic, request.releaseRequest);
+            this.processReleaseRequest(requestId, responseTopic, request.releaseRequest);
         } else {
             logger.error('[%s] Unknown request recevied!', requestId);
         }
@@ -68,7 +76,7 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r
             this.sendResponse(requestId, responseTopic, scriptId, compileResponse);
         },
         (err) => {
-            var compileResponse = createCompileResponse(scriptId, false, js.JsInvokeErrorCode.COMPILATION_ERROR, err);
+            var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err);
             logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId);
             this.sendResponse(requestId, responseTopic, scriptId, compileResponse);
         }
@@ -96,9 +104,9 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re
                 (err) => {
                     var errorCode;
                     if (err.message.includes('Script execution timed out')) {
-                        errorCode = js.JsInvokeErrorCode.TIMEOUT_ERROR;
+                        errorCode = TIMEOUT_ERROR;
                     } else {
-                        errorCode = js.JsInvokeErrorCode.RUNTIME_ERROR;
+                        errorCode = RUNTIME_ERROR;
                     }
                     var invokeResponse = createInvokeResponse("", false, errorCode, err);
                     logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode);
@@ -107,8 +115,8 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re
             )
         },
         (err) => {
-            var invokeResponse = createInvokeResponse("", false, js.JsInvokeErrorCode.COMPILATION_ERROR, err);
-            logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, js.JsInvokeErrorCode.COMPILATION_ERROR);
+            var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err);
+            logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR);
             this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse);
         }
     );
@@ -127,15 +135,26 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r
 
 JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) {
     var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse);
-    var rawResponse = js.RemoteJsResponse.encode(remoteResponse).finish();
-    const message = new KeyedMessage(scriptId, rawResponse);
-    const payloads = [ { topic: responseTopic, messages: message, key: scriptId } ];
-    this.producer.send(payloads, function (err, data) {
-        if (err) {
-            logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message);
-            logger.error(err.stack);
+    var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8');
+    this.producer.send(
+        {
+            topic: responseTopic,
+            messages: [
+                {
+                    key: scriptId,
+                    value: rawResponse
+                }
+            ]
         }
-    });
+    ).then(
+        () => {},
+        (err) => {
+            if (err) {
+                logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message);
+                logger.error(err.stack);
+            }
+        }
+    );
 }
 
 JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) {
@@ -159,50 +178,42 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scrip
 
 function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) {
     const requestIdBits = Utils.UUIDToBits(requestId);
-    return js.RemoteJsResponse.create(
-        {
+    return {
             requestIdMSB: requestIdBits[0],
             requestIdLSB: requestIdBits[1],
             compileResponse: compileResponse,
             invokeResponse: invokeResponse,
             releaseResponse: releaseResponse
-        }
-    );
+    };
 }
 
 function createCompileResponse(scriptId, success, errorCode, err) {
     const scriptIdBits = Utils.UUIDToBits(scriptId);
-    return js.JsCompileResponse.create(
-        {
+    return  {
             errorCode: errorCode,
             success: success,
             errorDetails: parseJsErrorDetails(err),
             scriptIdMSB: scriptIdBits[0],
             scriptIdLSB: scriptIdBits[1]
-        }
-    );
+    };
 }
 
 function createInvokeResponse(result, success, errorCode, err) {
-    return js.JsInvokeResponse.create(
-        {
+    return  {
             errorCode: errorCode,
             success: success,
             errorDetails: parseJsErrorDetails(err),
             result: result
-        }
-    );
+    };
 }
 
 function createReleaseResponse(scriptId, success) {
     const scriptIdBits = Utils.UUIDToBits(scriptId);
-    return js.JsReleaseResponse.create(
-        {
+    return {
             success: success,
             scriptIdMSB: scriptIdBits[0],
             scriptIdLSB: scriptIdBits[1]
-        }
-    );
+    };
 }
 
 function parseJsErrorDetails(err) {
@@ -229,8 +240,4 @@ function getScriptId(request) {
     return Utils.toUUIDString(request.scriptIdMSB, request.scriptIdLSB);
 }
 
-function getRequestId(request) {
-    return Utils.toUUIDString(request.requestIdMSB, request.requestIdLSB);
-}
-
-module.exports = JsInvokeMessageProcessor;
+module.exports = JsInvokeMessageProcessor;

msa/js-executor/api/utils.js

@@ -18,16 +18,17 @@
 const Long = require('long'),
       uuidParse = require('uuid-parse');
 
-var logger = require('../config/logger')('Utils');
-
 exports.toUUIDString = function(mostSigBits, leastSigBits) {
     var msbBytes = Long.fromValue(mostSigBits, false).toBytes(false);
     var lsbBytes = Long.fromValue(leastSigBits, false).toBytes(false);
     var uuidBytes = msbBytes.concat(lsbBytes);
-    var buff = new Buffer(uuidBytes, 'utf8');
     return uuidParse.unparse(uuidBytes);
 }
 
+exports.UUIDFromBuffer = function(buf) {
+    return uuidParse.unparse(buf);
+}
+
 exports.UUIDToBits = function(uuidString) {
     const bytes = uuidParse.parse(uuidString);
     var msb = Long.fromBytes(bytes.slice(0,8), false, false).toString();

msa/js-executor/config/custom-environment-variables.yml

@@ -25,4 +25,5 @@ logger:
   filename: "LOGGER_FILENAME"
 
 script:
+  use_sandbox: "SCRIPT_USE_SANDBOX"
   script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY"

msa/js-executor/config/default.yml

@@ -26,4 +26,5 @@ logger:
   filename: "tb-js-executor-%DATE%.log"
 
 script:
+  use_sandbox: "true"
   script_body_trace_frequency: "1000"

msa/js-executor/config/logger.js

@@ -17,9 +17,24 @@ var config = require('config'),
     path = require('path'),
     DailyRotateFile = require('winston-daily-rotate-file');
 
+const { logLevel } = require('kafkajs');
 const { createLogger, format, transports } = require('winston');
 const { combine, timestamp, label, printf, splat } = format;
 
+const toWinstonLogLevel = level => {
+    switch(level) {
+        case logLevel.ERROR:
+        case logLevel.NOTHING:
+            return 'error'
+        case logLevel.WARN:
+            return 'warn'
+        case logLevel.INFO:
+            return 'info'
+        case logLevel.DEBUG:
+            return 'debug'
+    }
+}
+
 var loggerTransports = [];
 
 if (process.env.NODE_ENV !== 'production' || process.env.DOCKER_MODE === 'true') {
@@ -56,4 +71,33 @@ function _logger(moduleLabel) {
     });
 }
 
-module.exports = _logger;
+const KafkaJsWinstonLogCreator = logLevel => {
+    const logger = createLogger({
+        level: toWinstonLogLevel(logLevel),
+        format:combine(
+            splat(),
+            label({ label: 'kafkajs' }),
+            timestamp({format: 'YYYY-MM-DD HH:mm:ss,SSS'}),
+            printf(info => {
+                var res = `${info.timestamp} [${info.label}] ${info.level.toUpperCase()}: ${info.message}`;
+                if (info.extra) {
+                    res +=`: ${JSON.stringify(info.extra)}`;
+                }
+                return res;
+              }
+            )
+        ),
+        transports: loggerTransports
+    });
+
+    return ({ namespace, level, label, log }) => {
+        const { message, ...extra } = log;
+        logger.log({
+            level: toWinstonLogLevel(level),
+            message,
+            extra,
+        });
+    }
+}
+
+module.exports = {_logger, KafkaJsWinstonLogCreator};

msa/js-executor/package-lock.json

@@ -35,60 +35,6 @@
       "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==",
       "dev": true
     },
-    "@protobufjs/aspromise": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
-      "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78="
-    },
-    "@protobufjs/base64": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
-      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
-    },
-    "@protobufjs/codegen": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
-      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
-    },
-    "@protobufjs/eventemitter": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
-      "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A="
-    },
-    "@protobufjs/fetch": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
-      "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=",
-      "requires": {
-        "@protobufjs/aspromise": "^1.1.1",
-        "@protobufjs/inquire": "^1.1.0"
-      }
-    },
-    "@protobufjs/float": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
-      "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E="
-    },
-    "@protobufjs/inquire": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
-      "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik="
-    },
-    "@protobufjs/path": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
-      "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0="
-    },
-    "@protobufjs/pool": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
-      "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q="
-    },
-    "@protobufjs/utf8": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
-      "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
-    },
     "@types/events": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz",
@@ -106,11 +52,6 @@
         "@types/node": "*"
       }
     },
-    "@types/long": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz",
-      "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q=="
-    },
     "@types/minimatch": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
@@ -120,7 +61,8 @@
     "@types/node": {
       "version": "10.12.10",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz",
-      "integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w=="
+      "integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w==",
+      "dev": true
     },
     "abbrev": {
       "version": "1.1.1",
@@ -182,11 +124,6 @@
         }
       }
     },
-    "ansi-regex": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
-      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
-    },
     "ansi-styles": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
@@ -206,22 +143,6 @@
         "normalize-path": "^2.1.1"
       }
     },
-    "aproba": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
-      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
-      "optional": true
-    },
-    "are-we-there-yet": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
-      "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
-      "optional": true,
-      "requires": {
-        "delegates": "^1.0.0",
-        "readable-stream": "^2.0.6"
-      }
-    },
     "argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -331,7 +252,8 @@
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "dev": true
     },
     "base": {
       "version": "0.11.2",
@@ -397,39 +319,12 @@
         "tweetnacl": "^0.14.3"
       }
     },
-    "binary": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
-      "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
-      "requires": {
-        "buffers": "~0.1.1",
-        "chainsaw": "~0.1.0"
-      }
-    },
     "binary-extensions": {
       "version": "1.12.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.12.0.tgz",
       "integrity": "sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg==",
       "dev": true
     },
-    "bindings": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
-      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
-      "optional": true,
-      "requires": {
-        "file-uri-to-path": "1.0.0"
-      }
-    },
-    "bl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz",
-      "integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==",
-      "requires": {
-        "readable-stream": "^2.3.5",
-        "safe-buffer": "^5.1.1"
-      }
-    },
     "boxen": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz",
@@ -482,6 +377,7 @@
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
       "requires": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -516,53 +412,6 @@
         }
       }
     },
-    "buffer-alloc": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
-      "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
-      "optional": true,
-      "requires": {
-        "buffer-alloc-unsafe": "^1.1.0",
-        "buffer-fill": "^1.0.0"
-      }
-    },
-    "buffer-alloc-unsafe": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
-      "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==",
-      "optional": true
-    },
-    "buffer-crc32": {
-      "version": "0.2.13",
-      "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
-      "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
-    },
-    "buffer-fill": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
-      "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=",
-      "optional": true
-    },
-    "buffermaker": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.1.tgz",
-      "integrity": "sha512-IdnyU2jDHU65U63JuVQNTHiWjPRH0CS3aYd/WPaEwyX84rFdukhOduAVb1jwUScmb5X0JWPw8NZOrhoLMiyAHQ==",
-      "requires": {
-        "long": "1.1.2"
-      },
-      "dependencies": {
-        "long": {
-          "version": "1.1.2",
-          "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz",
-          "integrity": "sha1-6u9ZUcp1UdlpJrgtokLbnWso+1M="
-        }
-      }
-    },
-    "buffers": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
-      "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
-    },
     "byline": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
@@ -610,14 +459,6 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
       "dev": true
     },
-    "chainsaw": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
-      "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
-      "requires": {
-        "traverse": ">=0.3.0 <0.4"
-      }
-    },
     "chalk": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
@@ -650,12 +491,6 @@
         "upath": "^1.0.5"
       }
     },
-    "chownr": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz",
-      "integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==",
-      "optional": true
-    },
     "ci-info": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz",
@@ -691,11 +526,6 @@
       "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=",
       "dev": true
     },
-    "code-point-at": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
-      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
-    },
     "collection-visit": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
@@ -774,7 +604,8 @@
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
     },
     "config": {
       "version": "3.2.2",
@@ -798,11 +629,6 @@
         "xdg-basedir": "^3.0.0"
       }
     },
-    "console-control-strings": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
-      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
-    },
     "copy-descriptor": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
@@ -858,6 +684,7 @@
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
       "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+      "dev": true,
       "requires": {
         "ms": "2.0.0"
       }
@@ -868,19 +695,11 @@
       "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=",
       "dev": true
     },
-    "decompress-response": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
-      "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
-      "optional": true,
-      "requires": {
-        "mimic-response": "^1.0.0"
-      }
-    },
     "deep-extend": {
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
-      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+      "dev": true
     },
     "deep-is": {
       "version": "0.1.3",
@@ -935,23 +754,6 @@
       "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
       "dev": true
     },
-    "delegates": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
-      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=",
-      "optional": true
-    },
-    "denque": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
-      "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
-    },
-    "detect-libc": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
-      "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=",
-      "optional": true
-    },
     "diagnostics": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz",
@@ -1004,14 +806,6 @@
         "env-variable": "0.0.x"
       }
     },
-    "end-of-stream": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
-      "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
-      "requires": {
-        "once": "^1.4.0"
-      }
-    },
     "env-variable": {
       "version": "0.0.5",
       "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz",
@@ -1272,12 +1066,6 @@
         "moment": "^2.11.2"
       }
     },
-    "file-uri-to-path": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
-      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
-      "optional": true
-    },
     "fill-range": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
@@ -1343,12 +1131,6 @@
         "readable-stream": "^2.0.0"
       }
     },
-    "fs-constants": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
-      "optional": true
-    },
     "fs-extra": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-6.0.1.tgz",
@@ -1895,22 +1677,6 @@
         }
       }
     },
-    "gauge": {
-      "version": "2.7.4",
-      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
-      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
-      "optional": true,
-      "requires": {
-        "aproba": "^1.0.3",
-        "console-control-strings": "^1.0.0",
-        "has-unicode": "^2.0.0",
-        "object-assign": "^4.1.0",
-        "signal-exit": "^3.0.0",
-        "string-width": "^1.0.1",
-        "strip-ansi": "^3.0.1",
-        "wide-align": "^1.1.0"
-      }
-    },
     "get-stream": {
       "version": "3.0.0",
       "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
@@ -1932,12 +1698,6 @@
         "assert-plus": "^1.0.0"
       }
     },
-    "github-from-package": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
-      "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=",
-      "optional": true
-    },
     "glob": {
       "version": "7.1.4",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
@@ -2059,12 +1819,6 @@
       "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
       "dev": true
     },
-    "has-unicode": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
-      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
-      "optional": true
-    },
     "has-value": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
@@ -2150,7 +1904,8 @@
     "ini": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
+      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "dev": true
     },
     "into-stream": {
       "version": "5.1.0",
@@ -2262,14 +2017,6 @@
       "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
       "dev": true
     },
-    "is-fullwidth-code-point": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-      "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-      "requires": {
-        "number-is-nan": "^1.0.0"
-      }
-    },
     "is-glob": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
@@ -2453,42 +2200,12 @@
         "verror": "1.10.0"
       }
     },
-    "kafka-node": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-4.1.3.tgz",
-      "integrity": "sha512-C2WHksRCr7vIKmbxYaCk2c5Q1lnHIi6C0f3AioK3ARcRHGO9DpqErcoaS9d8PP62yzTnkYras+iAlmPsZHNSfw==",
-      "requires": {
-        "async": "^2.6.2",
-        "binary": "~0.3.0",
-        "bl": "^2.2.0",
-        "buffer-crc32": "~0.2.5",
-        "buffermaker": "~1.2.0",
-        "debug": "^2.1.3",
-        "denque": "^1.3.0",
-        "lodash": "^4.17.4",
-        "minimatch": "^3.0.2",
-        "nested-error-stacks": "^2.0.0",
-        "optional": "^0.1.3",
-        "retry": "^0.10.1",
-        "snappy": "^6.0.1",
-        "uuid": "^3.0.0"
-      },
-      "dependencies": {
-        "async": {
-          "version": "2.6.3",
-          "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
-          "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
-          "requires": {
-            "lodash": "^4.17.14"
-          },
-          "dependencies": {
-            "lodash": {
-              "version": "4.17.15",
-              "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-              "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
-            }
-          }
-        }
+    "kafkajs": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-1.11.0.tgz",
+      "integrity": "sha512-dLRCcFIBygZucR+e8U2ZqH2wgMrAu114K0szUyUseJoeOii3cG5bHZPIdqKecXxI6begPVCfGS3R0nJY4zHW2A==",
+      "requires": {
+        "long": "^4.0.0"
       }
     },
     "kind-of": {
@@ -2641,16 +2358,11 @@
         "mime-db": "1.40.0"
       }
     },
-    "mimic-response": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
-      "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
-      "optional": true
-    },
     "minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
       "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "dev": true,
       "requires": {
         "brace-expansion": "^1.1.7"
       }
@@ -2685,6 +2397,7 @@
       "version": "0.5.1",
       "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
       "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "dev": true,
       "requires": {
         "minimist": "0.0.8"
       },
@@ -2692,7 +2405,8 @@
         "minimist": {
           "version": "0.0.8",
           "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-          "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+          "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
+          "dev": true
         }
       }
     },
@@ -2704,7 +2418,8 @@
     "ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+      "dev": true
     },
     "multistream": {
       "version": "2.1.1",
@@ -2742,26 +2457,6 @@
         "to-regex": "^3.0.1"
       }
     },
-    "napi-build-utils": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.1.tgz",
-      "integrity": "sha512-boQj1WFgQH3v4clhu3mTNfP+vOBxorDlE8EKiMjUlLG3C4qAESnn9AxIOkFgTR2c9LtzNjPrjS60cT27ZKBhaA==",
-      "optional": true
-    },
-    "nested-error-stacks": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz",
-      "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug=="
-    },
-    "node-abi": {
-      "version": "2.9.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.9.0.tgz",
-      "integrity": "sha512-jmEOvv0eanWjhX8dX1pmjb7oJl1U1oR4FOh0b2GnvALwSYoOdU7sj+kLDSAyjo4pfC9aj/IxkloxdLJQhSSQBA==",
-      "optional": true,
-      "requires": {
-        "semver": "^5.4.1"
-      }
-    },
     "nodemon": {
       "version": "1.18.7",
       "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.18.7.tgz",
@@ -2797,12 +2492,6 @@
         }
       }
     },
-    "noop-logger": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
-      "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=",
-      "optional": true
-    },
     "nopt": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz",
@@ -2830,35 +2519,12 @@
         "path-key": "^2.0.0"
       }
     },
-    "npmlog": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
-      "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
-      "optional": true,
-      "requires": {
-        "are-we-there-yet": "~1.1.2",
-        "console-control-strings": "~1.1.0",
-        "gauge": "~2.7.3",
-        "set-blocking": "~2.0.0"
-      }
-    },
-    "number-is-nan": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
-      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
-    },
     "oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
       "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
       "dev": true
     },
-    "object-assign": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=",
-      "optional": true
-    },
     "object-copy": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
@@ -2917,6 +2583,7 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
       "requires": {
         "wrappy": "1"
       }
@@ -2926,11 +2593,6 @@
       "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz",
       "integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4="
     },
-    "optional": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz",
-      "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw=="
-    },
     "optionator": {
       "version": "0.8.2",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
@@ -2945,12 +2607,6 @@
         "wordwrap": "~1.0.0"
       }
     },
-    "os-homedir": {
-      "version": "1.0.2",
-      "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
-      "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=",
-      "optional": true
-    },
     "os-tmpdir": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
@@ -3127,38 +2783,6 @@
       "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=",
       "dev": true
     },
-    "prebuild-install": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz",
-      "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==",
-      "optional": true,
-      "requires": {
-        "detect-libc": "^1.0.3",
-        "expand-template": "^2.0.3",
-        "github-from-package": "0.0.0",
-        "minimist": "^1.2.0",
-        "mkdirp": "^0.5.1",
-        "napi-build-utils": "^1.0.1",
-        "node-abi": "^2.7.0",
-        "noop-logger": "^0.1.1",
-        "npmlog": "^4.0.1",
-        "os-homedir": "^1.0.1",
-        "pump": "^2.0.1",
-        "rc": "^1.2.7",
-        "simple-get": "^2.7.0",
-        "tar-fs": "^1.13.0",
-        "tunnel-agent": "^0.6.0",
-        "which-pm-runs": "^1.0.0"
-      },
-      "dependencies": {
-        "expand-template": {
-          "version": "2.0.3",
-          "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
-          "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
-          "optional": true
-        }
-      }
-    },
     "prelude-ls": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
@@ -3182,26 +2806,6 @@
       "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
       "dev": true
     },
-    "protobufjs": {
-      "version": "6.8.8",
-      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz",
-      "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==",
-      "requires": {
-        "@protobufjs/aspromise": "^1.1.2",
-        "@protobufjs/base64": "^1.1.2",
-        "@protobufjs/codegen": "^2.0.4",
-        "@protobufjs/eventemitter": "^1.1.0",
-        "@protobufjs/fetch": "^1.1.0",
-        "@protobufjs/float": "^1.0.2",
-        "@protobufjs/inquire": "^1.1.0",
-        "@protobufjs/path": "^1.1.2",
-        "@protobufjs/pool": "^1.1.0",
-        "@protobufjs/utf8": "^1.1.0",
-        "@types/long": "^4.0.0",
-        "@types/node": "^10.1.0",
-        "long": "^4.0.0"
-      }
-    },
     "pseudomap": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
@@ -3220,16 +2824,6 @@
       "integrity": "sha512-vL6NLxNHzkNTjGJUpMm5PLC+94/0tTlC1vkP9bdU0pOHih+EujMjgMTwfZopZvHWRFbqJ5Y73OMoau50PewDDA==",
       "dev": true
     },
-    "pump": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
-      "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
-      "optional": true,
-      "requires": {
-        "end-of-stream": "^1.1.0",
-        "once": "^1.3.1"
-      }
-    },
     "punycode": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
@@ -3246,6 +2840,7 @@
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
       "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "dev": true,
       "requires": {
         "deep-extend": "^0.6.0",
         "ini": "~1.3.0",
@@ -3389,11 +2984,6 @@
       "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==",
       "dev": true
     },
-    "retry": {
-      "version": "0.10.1",
-      "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz",
-      "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q="
-    },
     "safe-buffer": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@@ -3428,12 +3018,6 @@
         "semver": "^5.0.3"
       }
     },
-    "set-blocking": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
-      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
-      "optional": true
-    },
     "set-value": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
@@ -3475,24 +3059,8 @@
     "signal-exit": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
-      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
-    },
-    "simple-concat": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz",
-      "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=",
-      "optional": true
-    },
-    "simple-get": {
-      "version": "2.8.1",
-      "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz",
-      "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==",
-      "optional": true,
-      "requires": {
-        "decompress-response": "^3.3.0",
-        "once": "^1.3.1",
-        "simple-concat": "^1.0.0"
-      }
+      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
+      "dev": true
     },
     "simple-swizzle": {
       "version": "0.2.2",
@@ -3615,25 +3183,6 @@
         }
       }
     },
-    "snappy": {
-      "version": "6.2.3",
-      "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.2.3.tgz",
-      "integrity": "sha512-HZpVoIxMfQ4fL3iDuMdI1R5xycw1o9YDCAndTKZCY/EHRoKFvzwplttuBBVGeEg2fd1hYiwAXos/sM24W7N1LA==",
-      "optional": true,
-      "requires": {
-        "bindings": "^1.3.1",
-        "nan": "^2.14.0",
-        "prebuild-install": "^5.2.2"
-      },
-      "dependencies": {
-        "nan": {
-          "version": "2.14.0",
-          "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
-          "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==",
-          "optional": true
-        }
-      }
-    },
     "source-map": {
       "version": "0.5.7",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
@@ -3725,16 +3274,6 @@
         "readable-stream": "^2.1.4"
       }
     },
-    "string-width": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-      "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-      "requires": {
-        "code-point-at": "^1.0.0",
-        "is-fullwidth-code-point": "^1.0.0",
-        "strip-ansi": "^3.0.0"
-      }
-    },
     "string_decoder": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
@@ -3743,14 +3282,6 @@
         "safe-buffer": "~5.1.0"
       }
     },
-    "strip-ansi": {
-      "version": "3.0.1",
-      "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
-      "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
-      "requires": {
-        "ansi-regex": "^2.0.0"
-      }
-    },
     "strip-eof": {
       "version": "1.0.0",
       "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
@@ -3760,7 +3291,8 @@
     "strip-json-comments": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
-      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
+      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
+      "dev": true
     },
     "supports-color": {
       "version": "5.5.0",
@@ -3771,57 +3303,6 @@
         "has-flag": "^3.0.0"
       }
     },
-    "tar-fs": {
-      "version": "1.16.3",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz",
-      "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==",
-      "optional": true,
-      "requires": {
-        "chownr": "^1.0.1",
-        "mkdirp": "^0.5.1",
-        "pump": "^1.0.0",
-        "tar-stream": "^1.1.2"
-      },
-      "dependencies": {
-        "pump": {
-          "version": "1.0.3",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz",
-          "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==",
-          "optional": true,
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        }
-      }
-    },
-    "tar-stream": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
-      "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
-      "optional": true,
-      "requires": {
-        "bl": "^1.0.0",
-        "buffer-alloc": "^1.2.0",
-        "end-of-stream": "^1.0.0",
-        "fs-constants": "^1.0.0",
-        "readable-stream": "^2.3.0",
-        "to-buffer": "^1.1.1",
-        "xtend": "^4.0.0"
-      },
-      "dependencies": {
-        "bl": {
-          "version": "1.2.2",
-          "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
-          "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
-          "optional": true,
-          "requires": {
-            "readable-stream": "^2.3.5",
-            "safe-buffer": "^5.1.1"
-          }
-        }
-      }
-    },
     "term-size": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz",
@@ -3848,12 +3329,6 @@
       "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=",
       "dev": true
     },
-    "to-buffer": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
-      "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==",
-      "optional": true
-    },
     "to-object-path": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz",
@@ -3923,11 +3398,6 @@
         }
       }
     },
-    "traverse": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
-      "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
-    },
     "triple-beam": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
@@ -3937,6 +3407,7 @@
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
       "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
+      "dev": true,
       "requires": {
         "safe-buffer": "^5.0.1"
       }
@@ -4140,7 +3611,8 @@
     "uuid": {
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
-      "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
+      "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
+      "dev": true
     },
     "uuid-parse": {
       "version": "1.0.0",
@@ -4167,21 +3639,6 @@
         "isexe": "^2.0.0"
       }
     },
-    "which-pm-runs": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz",
-      "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=",
-      "optional": true
-    },
-    "wide-align": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
-      "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
-      "optional": true,
-      "requires": {
-        "string-width": "^1.0.2 || 2"
-      }
-    },
     "widest-line": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz",
@@ -4281,7 +3738,8 @@
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
     },
     "write-file-atomic": {
       "version": "2.3.0",
@@ -4300,12 +3758,6 @@
       "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=",
       "dev": true
     },
-    "xtend": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
-      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
-      "optional": true
-    },
     "yallist": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",

msa/js-executor/package.json

@@ -6,18 +6,16 @@
   "main": "server.js",
   "bin": "server.js",
   "scripts": {
-    "build-proto": "pbjs -t static-module -w commonjs -o ./api/jsinvoke.proto.js ../../application/src/main/proto/jsinvoke.proto",
-    "install": "npm run build-proto && pkg -t node10-linux-x64,node10-win-x64 --out-path ./target . && node install.js",
+    "install": "pkg -t node10-linux-x64,node10-win-x64 --out-path ./target . && node install.js",
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "npm run build-proto && nodemon server.js",
-    "start-prod": "npm run build-proto && NODE_ENV=production nodemon server.js"
+    "start": "nodemon server.js",
+    "start-prod": "NODE_ENV=production nodemon server.js"
   },
   "dependencies": {
     "config": "^3.2.2",
     "js-yaml": "^3.12.0",
-    "kafka-node": "^4.1.3",
+    "kafkajs": "^1.11.0",
     "long": "^4.0.0",
-    "protobufjs": "^6.8.8",
     "uuid-parse": "^1.0.0",
     "winston": "^3.0.0",
     "winston-daily-rotate-file": "^3.2.1"

msa/js-executor/server.js

@@ -13,14 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+const { logLevel, Kafka } = require('kafkajs');
+
 const config = require('config'),
-      kafka = require('kafka-node'),
-      ConsumerGroup = kafka.ConsumerGroup,
-      Producer = kafka.Producer,
       JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'),
-      logger = require('./config/logger')('main');
+      logger = require('./config/logger')._logger('main'),
+      KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator;
 
 var kafkaClient;
+var consumer;
+var producer;
 
 (async() => {
     try {
@@ -32,49 +35,24 @@ var kafkaClient;
         logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
         logger.info('Kafka Requests Topic: %s', kafkaRequestTopic);
 
-        kafkaClient = new kafka.KafkaClient({kafkaHost: kafkaBootstrapServers});
-
-        var consumer = new ConsumerGroup(
-            {
-                kafkaHost: kafkaBootstrapServers,
-                groupId: 'js-executor-group',
-                autoCommit: true,
-                encoding: 'buffer'
-            },
-            kafkaRequestTopic
-        );
-
-        consumer.on('error', (err) => {
-            logger.error('Unexpected kafka consumer error: %s', err.message);
-            logger.error(err.stack);
-        });
-
-        consumer.on('offsetOutOfRange', (err) => {
-            logger.error('Offset out of range error: %s', err.message);
-            logger.error(err.stack);
-        });
-
-        consumer.on('rebalancing', () => {
-            logger.info('Rebalancing event received.');
-        })
-
-        consumer.on('rebalanced', () => {
-            logger.info('Rebalanced event received.');
-        });
-
-        var producer = new Producer(kafkaClient);
-        producer.on('error', (err) => {
-            logger.error('Unexpected kafka producer error: %s', err.message);
-            logger.error(err.stack);
+        kafkaClient = new Kafka({
+            brokers: kafkaBootstrapServers.split(','),
+            logLevel: logLevel.INFO,
+            logCreator: KafkaJsWinstonLogCreator
         });
 
-        var messageProcessor = new JsInvokeMessageProcessor(producer);
+        consumer = kafkaClient.consumer({ groupId: 'js-executor-group' });
+        producer = kafkaClient.producer();
+        const messageProcessor = new JsInvokeMessageProcessor(producer);
+        await consumer.connect();
+        await producer.connect();
+        await consumer.subscribe({ topic: kafkaRequestTopic});
 
-        producer.on('ready', () => {
-            consumer.on('message', (message) => {
+        logger.info('Started ThingsBoard JavaScript Executor Microservice.');
+        await consumer.run({
+            eachMessage: async ({ topic, partition, message }) => {
                 messageProcessor.onJsInvokeMessage(message);
-            });
-            logger.info('Started ThingsBoard JavaScript Executor Microservice.');
+            },
         });
 
     } catch (e) {
@@ -84,21 +62,41 @@ var kafkaClient;
     }
 })();
 
-process.on('exit', function () {
+process.on('exit', () => {
     exit(0);
 });
 
-function exit(status) {
+async function exit(status) {
     logger.info('Exiting with status: %d ...', status);
-    if (kafkaClient) {
-        logger.info('Stopping Kafka Client...');
-        var _kafkaClient = kafkaClient;
-        kafkaClient = null;
-        _kafkaClient.close(() => {
-            logger.info('Kafka Client stopped.');
+    if (consumer) {
+        logger.info('Stopping Kafka Consumer...');
+        var _consumer = consumer;
+        consumer = null;
+        try {
+            await _consumer.disconnect();
+            logger.info('Kafka Consumer stopped.');
+            await disconnectProducer();
             process.exit(status);
-        });
+        } catch (e) {
+            logger.info('Kafka Consumer stop error.');
+            await disconnectProducer();
+            process.exit(status);
+        }
     } else {
         process.exit(status);
     }
 }
+
+async function disconnectProducer() {
+    if (producer) {
+        logger.info('Stopping Kafka Producer...');
+        var _producer = producer;
+        producer = null;
+        try {
+            await _producer.disconnect();
+            logger.info('Kafka Producer stopped.');
+        } catch (e) {
+            logger.info('Kafka Producer stop error.');
+        }
+    }
+}