From 97a0a786b76693e703e1000e865e22a7b0a2c3d3 Mon Sep 17 00:00:00 2001 From: vparomskiy Date: Wed, 12 Dec 2018 19:03:17 +0200 Subject: [PATCH 1/3] aggregation for numeric data types should process both types - double and long --- .../AggregatePartitionsFunction.java | 63 +++++++++++-------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java index ac5ee640c1..b5ebb10922 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java @@ -79,7 +79,7 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct } private void processResultSetRow(Row row, AggregationResult aggResult) { - long curCount; + long curCount = 0L; Long curLValue = null; Double curDValue = null; @@ -91,14 +91,17 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct long boolCount = row.getLong(BOOL_CNT_POS); long strCount = row.getLong(STR_CNT_POS); - if (longCount > 0) { - aggResult.dataType = DataType.LONG; - curCount = longCount; - curLValue = getLongValue(row); - } else if (doubleCount > 0) { - aggResult.dataType = DataType.DOUBLE; - curCount = doubleCount; - curDValue = getDoubleValue(row); + if (longCount > 0 || doubleCount > 0) { + if (longCount > 0) { + aggResult.dataType = DataType.LONG; + curCount += longCount; + curLValue = getLongValue(row); + } + if (doubleCount > 0) { + aggResult.dataType = DataType.DOUBLE; + curCount += doubleCount; + curDValue = getDoubleValue(row); + } } else if (boolCount > 0) { aggResult.dataType = DataType.BOOLEAN; curCount = boolCount; @@ -126,16 +129,20 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct aggResult.count += curCount; if (curDValue != null) { aggResult.dValue = aggResult.dValue == null ? curDValue : aggResult.dValue + curDValue; - } else if (curLValue != null) { + } + if (curLValue != null) { aggResult.lValue = aggResult.lValue == null ? curLValue : aggResult.lValue + curLValue; } } private void processMinAggregation(AggregationResult aggResult, Long curLValue, Double curDValue, Boolean curBValue, String curSValue) { - if (curDValue != null) { - aggResult.dValue = aggResult.dValue == null ? curDValue : Math.min(aggResult.dValue, curDValue); - } else if (curLValue != null) { - aggResult.lValue = aggResult.lValue == null ? curLValue : Math.min(aggResult.lValue, curLValue); + if (curDValue != null || curLValue != null) { + if (curDValue != null) { + aggResult.dValue = aggResult.dValue == null ? curDValue : Math.min(aggResult.dValue, curDValue); + } + if (curLValue != null) { + aggResult.lValue = aggResult.lValue == null ? curLValue : Math.min(aggResult.lValue, curLValue); + } } else if (curBValue != null) { aggResult.bValue = aggResult.bValue == null ? curBValue : aggResult.bValue && curBValue; } else if (curSValue != null && (aggResult.sValue == null || curSValue.compareTo(aggResult.sValue) < 0)) { @@ -144,10 +151,13 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct } private void processMaxAggregation(AggregationResult aggResult, Long curLValue, Double curDValue, Boolean curBValue, String curSValue) { - if (curDValue != null) { - aggResult.dValue = aggResult.dValue == null ? curDValue : Math.max(aggResult.dValue, curDValue); - } else if (curLValue != null) { - aggResult.lValue = aggResult.lValue == null ? curLValue : Math.max(aggResult.lValue, curLValue); + if (curDValue != null || curLValue != null) { + if (curDValue != null) { + aggResult.dValue = aggResult.dValue == null ? curDValue : Math.max(aggResult.dValue, curDValue); + } + if (curLValue != null) { + aggResult.lValue = aggResult.lValue == null ? curLValue : Math.max(aggResult.lValue, curLValue); + } } else if (curBValue != null) { aggResult.bValue = aggResult.bValue == null ? curBValue : aggResult.bValue || curBValue; } else if (curSValue != null && (aggResult.sValue == null || curSValue.compareTo(aggResult.sValue) > 0)) { @@ -211,20 +221,19 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct private Optional processAvgOrSumResult(AggregationResult aggResult) { if (aggResult.count == 0 || (aggResult.dataType == DataType.DOUBLE && aggResult.dValue == null) || (aggResult.dataType == DataType.LONG && aggResult.lValue == null)) { return Optional.empty(); - } else if (aggResult.dataType == DataType.DOUBLE) { - return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? aggResult.dValue : (aggResult.dValue / aggResult.count)))); - } else if (aggResult.dataType == DataType.LONG) { - return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggregation == Aggregation.SUM ? aggResult.lValue : (aggResult.lValue / aggResult.count)))); + } else if (aggResult.dataType == DataType.DOUBLE || aggResult.dataType == DataType.LONG) { + double sum = Optional.ofNullable(aggResult.dValue).orElse(0.0d) + Optional.ofNullable(aggResult.lValue).orElse(0L); + return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? sum : (sum / aggResult.count)))); } return Optional.empty(); } private Optional processMinOrMaxResult(AggregationResult aggResult) { - if (aggResult.dataType == DataType.DOUBLE) { - return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggResult.dValue))); - } else if (aggResult.dataType == DataType.LONG) { - return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggResult.lValue))); - } else if (aggResult.dataType == DataType.STRING) { + if (aggResult.dataType == DataType.DOUBLE || aggResult.dataType == DataType.LONG) { + double currentD = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.dValue).orElse(Double.MAX_VALUE) : Optional.ofNullable(aggResult.dValue).orElse(Double.MIN_VALUE); + double currentL = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.lValue).orElse(Long.MAX_VALUE) : Optional.ofNullable(aggResult.lValue).orElse(Long.MIN_VALUE); + return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.MIN ? Math.min(currentD, currentL) : Math.max(currentD, currentL)))); + } else if (aggResult.dataType == DataType.STRING) { return Optional.of(new BasicTsKvEntry(ts, new StringDataEntry(key, aggResult.sValue))); } else { return Optional.of(new BasicTsKvEntry(ts, new BooleanDataEntry(key, aggResult.bValue))); From 094ef761cd18b5423d6b6f6ecbf441ca44d1667a Mon Sep 17 00:00:00 2001 From: vparomskiy Date: Thu, 31 Jan 2019 16:07:14 +0200 Subject: [PATCH 2/3] Merge remote-tracking branch 'upstream/master' # Conflicts: # dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java --- .../timeseries/AggregatePartitionsFunction.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java index 10e5c860e7..ea1e8f1c1b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java @@ -98,10 +98,7 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct curLValue = getLongValue(row); } if (doubleCount > 0) { -<<<<<<< HEAD -======= aggResult.hasDouble = true; ->>>>>>> upstream/master aggResult.dataType = DataType.DOUBLE; curCount += doubleCount; curDValue = getDoubleValue(row); @@ -226,28 +223,18 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct if (aggResult.count == 0 || (aggResult.dataType == DataType.DOUBLE && aggResult.dValue == null) || (aggResult.dataType == DataType.LONG && aggResult.lValue == null)) { return Optional.empty(); } else if (aggResult.dataType == DataType.DOUBLE || aggResult.dataType == DataType.LONG) { -<<<<<<< HEAD - double sum = Optional.ofNullable(aggResult.dValue).orElse(0.0d) + Optional.ofNullable(aggResult.lValue).orElse(0L); - return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? sum : (sum / aggResult.count)))); -======= if(aggregation == Aggregation.AVG || aggResult.hasDouble) { double sum = Optional.ofNullable(aggResult.dValue).orElse(0.0d) + Optional.ofNullable(aggResult.lValue).orElse(0L); return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? sum : (sum / aggResult.count)))); } else { return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggregation == Aggregation.SUM ? aggResult.lValue : (aggResult.lValue / aggResult.count)))); } ->>>>>>> upstream/master } return Optional.empty(); } private Optional processMinOrMaxResult(AggregationResult aggResult) { if (aggResult.dataType == DataType.DOUBLE || aggResult.dataType == DataType.LONG) { -<<<<<<< HEAD - double currentD = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.dValue).orElse(Double.MAX_VALUE) : Optional.ofNullable(aggResult.dValue).orElse(Double.MIN_VALUE); - double currentL = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.lValue).orElse(Long.MAX_VALUE) : Optional.ofNullable(aggResult.lValue).orElse(Long.MIN_VALUE); - return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.MIN ? Math.min(currentD, currentL) : Math.max(currentD, currentL)))); -======= if(aggResult.hasDouble) { double currentD = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.dValue).orElse(Double.MAX_VALUE) : Optional.ofNullable(aggResult.dValue).orElse(Double.MIN_VALUE); double currentL = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.lValue).orElse(Long.MAX_VALUE) : Optional.ofNullable(aggResult.lValue).orElse(Long.MIN_VALUE); @@ -255,7 +242,6 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct } else { return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggResult.lValue))); } ->>>>>>> upstream/master } else if (aggResult.dataType == DataType.STRING) { return Optional.of(new BasicTsKvEntry(ts, new StringDataEntry(key, aggResult.sValue))); } else { From e9e10d74b89b358a028bd1a3f500928aab1c817d Mon Sep 17 00:00:00 2001 From: vparomskiy Date: Thu, 31 Jan 2019 16:09:50 +0200 Subject: [PATCH 3/3] cache webpack resources and run loaders in concurrent mode --- ui/package.json | 6 ++++-- ui/webpack.config.dev.js | 33 +++++++++++++++++++++++++++++---- ui/webpack.config.prod.js | 30 +++++++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/ui/package.json b/ui/package.json index 24f4bf6e65..a9ec3e0d6c 100644 --- a/ui/package.json +++ b/ui/package.json @@ -11,7 +11,7 @@ ], "scripts": { "start": "babel-node --max_old_space_size=4096 server.js", - "build": "cross-env NODE_ENV=production webpack -p" + "build": "cross-env NODE_ENV=production webpack" }, "dependencies": { "@flowjs/ng-flow": "^2.7.1", @@ -136,7 +136,9 @@ "webpack-dev-middleware": "^1.6.1", "webpack-dev-server": "^1.15.1", "webpack-hot-middleware": "^2.12.2", - "webpack-material-design-icons": "^0.1.0" + "webpack-material-design-icons": "^0.1.0", + "uglifyjs-webpack-plugin": "^1.3.0", + "happypack": "^5.0.1" }, "engine": "node >= 5.9.0", "nyc": { diff --git a/ui/webpack.config.dev.js b/ui/webpack.config.dev.js index cfff811b23..bea9275315 100644 --- a/ui/webpack.config.dev.js +++ b/ui/webpack.config.dev.js @@ -18,13 +18,16 @@ const HtmlWebpackPlugin = require('html-webpack-plugin'); const ExtractTextPlugin = require('extract-text-webpack-plugin'); const CopyWebpackPlugin = require('copy-webpack-plugin'); -const StyleLintPlugin = require('stylelint-webpack-plugin') +const StyleLintPlugin = require('stylelint-webpack-plugin'); const webpack = require('webpack'); const path = require('path'); const dirTree = require('directory-tree'); const jsonminify = require("jsonminify"); +const UglifyJsPlugin = require('uglifyjs-webpack-plugin'); +const HappyPack = require('happypack'); + const PUBLIC_RESOURCE_PATH = '/'; var langs = []; @@ -34,6 +37,9 @@ dirTree('./src/app/locale/', {extensions:/\.json$/}, (item) => { langs.push(item.name.slice(item.name.lastIndexOf('-') + 1, -5)); }); + +var happyThreadPool = HappyPack.ThreadPool({ size: 3 }); + /* devtool: 'cheap-module-eval-source-map', */ module.exports = { @@ -93,6 +99,25 @@ module.exports = { PUBLIC_PATH: JSON.stringify(PUBLIC_RESOURCE_PATH), SUPPORTED_LANGS: JSON.stringify(langs) }), + new UglifyJsPlugin({ + cache: true, + parallel: true + }), + new HappyPack({ + threadPool: happyThreadPool, + id: 'cached-babel', + loaders: ["babel-loader?cacheDirectory=true"] + }), + new HappyPack({ + threadPool: happyThreadPool, + id: 'eslint', + loaders: ["eslint-loader?{parser: 'babel-eslint'}"] + }), + new HappyPack({ + threadPool: happyThreadPool, + id: 'ng-annotate-and-cached-babel-loader', + loaders: ['ng-annotate', 'babel-loader?cacheDirectory=true'] + }) ], node: { tls: "empty", @@ -102,19 +127,19 @@ module.exports = { loaders: [ { test: /\.jsx$/, - loader: 'babel', + loader: 'happypack/loader?id=cached-babel', exclude: /node_modules/, include: __dirname, }, { test: /\.js$/, - loaders: ['ng-annotate', 'babel'], + loaders: ['happypack/loader?id=ng-annotate-and-cached-babel-loader'], exclude: /node_modules/, include: __dirname, }, { test: /\.js$/, - loader: "eslint-loader?{parser: 'babel-eslint'}", + loader: 'happypack/loader?id=eslint', exclude: /node_modules|vendor/, include: __dirname, }, diff --git a/ui/webpack.config.prod.js b/ui/webpack.config.prod.js index a442590d8b..c0809fe253 100644 --- a/ui/webpack.config.prod.js +++ b/ui/webpack.config.prod.js @@ -23,6 +23,8 @@ const webpack = require('webpack'); const path = require('path'); const dirTree = require('directory-tree'); const jsonminify = require("jsonminify"); +const UglifyJsPlugin = require('uglifyjs-webpack-plugin'); +const HappyPack = require('happypack'); const PUBLIC_RESOURCE_PATH = '/static/'; @@ -33,6 +35,8 @@ dirTree('./src/app/locale/', {extensions:/\.json$/}, (item) => { langs.push(item.name.slice(item.name.lastIndexOf('-') + 1, -5)); }); +var happyThreadPool = HappyPack.ThreadPool({ size: 3 }); + module.exports = { devtool: 'source-map', entry: [ @@ -95,6 +99,25 @@ module.exports = { test: /\.js$|\.css$|\.svg$|\.ttf$|\.woff$|\.woff2|\.eot$\.json$/, threshold: 10240, minRatio: 0.8 + }), + new UglifyJsPlugin({ + cache: true, + parallel: true + }), + new HappyPack({ + threadPool: happyThreadPool, + id: 'cached-babel', + loaders: ["babel-loader?cacheDirectory=true"] + }), + new HappyPack({ + threadPool: happyThreadPool, + id: 'eslint', + loaders: ["eslint-loader?{parser: 'babel-eslint'}"] + }), + new HappyPack({ + threadPool: happyThreadPool, + id: 'ng-annotate-and-cached-babel-loader', + loaders: ['ng-annotate', 'babel-loader?cacheDirectory=true'] }) ], node: { @@ -105,19 +128,20 @@ module.exports = { loaders: [ { test: /\.jsx$/, - loader: 'babel', + loader: 'happypack/loader?id=cached-babel', exclude: /node_modules/, include: __dirname, }, { test: /\.js$/, - loaders: ['ng-annotate', 'babel'], + loaders: ['happypack/loader?id=ng-annotate-and-cached-babel-loader'], exclude: /node_modules/, include: __dirname, }, { test: /\.js$/, - loader: "eslint-loader?{parser: 'babel-eslint'}", + loaders: ['happypack/loader?id=eslint'], + // loader: "eslint-loader?{parser: 'babel-eslint'}", exclude: /node_modules|vendor/, include: __dirname, },