From 1dd23e3ff22217ba88e6de941f642e5c89c16030 Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 15:33:52 +0800 Subject: [PATCH 1/4] improve log config. --- bench/main.go | 7 ++- conf.sample.yml | 11 ++-- dtmcli/logger/log.go | 114 ++++++++++++++++++++++++++++------- dtmcli/logger/logger_test.go | 43 ++++++------- dtmsvr/config/config.go | 11 ++-- dtmsvr/svr.go | 16 ++--- test/main_test.go | 2 +- 7 files changed, 135 insertions(+), 69 deletions(-) diff --git a/bench/main.go b/bench/main.go index 50e488b..11d49b9 100644 --- a/bench/main.go +++ b/bench/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "strings" "github.com/dtm-labs/dtm/bench/svr" "github.com/dtm-labs/dtm/dtmcli" @@ -33,7 +34,11 @@ func main() { } logger.Infof("starting bench server") config.MustLoadConfig("") - logger.InitLog(conf.Log.Level) + var outputs []string + if len(conf.Log.Outputs) != 0 { + outputs = strings.Split(conf.Log.Outputs, "|") + } + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) if busi.BusiConf.Driver != "" { dtmcli.SetCurrentDBType(busi.BusiConf.Driver) svr.PrepareBenchDB() diff --git a/conf.sample.yml b/conf.sample.yml index 5c046ed..71d56d7 100644 --- a/conf.sample.yml +++ b/conf.sample.yml @@ -49,13 +49,10 @@ # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm # Log: -# Level: 'info' # default: info. can be debug|info|warn|error -# Output: 'console' # default: console. can be console|file -# FileName: '/tmp/dtm.log' # default: /tmp/dtm.log. -# FileMaxSize: 10 # default: 10, unit: MB. -# FileMaxBackups: 5 # default: 5. -# FileMaxAge: 30 # default: 30, unit: days. -# FileCompress: 0 # default: 0. can by 0|1, means false|true +# Level: 'info' # default: info. can be debug|info|warn|error +# Outputs: '' # default: stdout, split by |, you can append files to Outputs if need. example:'stdout|/tmp/test.log' +# LogRotationEnable: 0 # default: 0 +# LogRotationConfigJson: '' # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}' # HttpPort: 36789 # GrpcPort: 36790 diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index e2ef110..0ad6e9f 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -1,6 +1,8 @@ package logger import ( + "encoding/json" + "errors" "fmt" "log" "net/url" @@ -15,8 +17,14 @@ import ( var logger Logger +const ( + DefaultLogOutput = "default" + StdErrLogOutput = "stderr" + StdOutLogOutput = "stdout" +) + func init() { - InitLog(os.Getenv("LOG_LEVEL")) + InitLog(os.Getenv("LOG_LEVEL"), nil, 0, "") } // Logger logger interface @@ -27,14 +35,6 @@ type Logger interface { Errorf(format string, args ...interface{}) } -type lumberjackSink struct { - *lumberjack.Logger -} - -func (lumberjackSink) Sync() error { - return nil -} - // WithLogger replaces default logger func WithLogger(log Logger) { logger = log @@ -42,27 +42,97 @@ func WithLogger(log Logger) { // InitLog is an initialization for a logger // level can be: debug info warn error -func InitLog(level string) { +func InitLog(level string, outputs []string, logRotationEnable int64, logRotateConfigJSON string) { + if len(outputs) == 0 { + outputs = []string{DefaultLogOutput} + } + + // parse outputs + outputPaths := make([]string, 0) + for _, v := range outputs { + switch v { + case DefaultLogOutput: + outputPaths = append(outputPaths, StdOutLogOutput) + + case StdErrLogOutput: + outputPaths = append(outputPaths, StdErrLogOutput) + + case StdOutLogOutput: + outputPaths = append(outputPaths, StdOutLogOutput) + + default: + var path string + if logRotationEnable != 0 { + // append rotate scheme to logs managed by lumberjack log rotation + if v[0:1] == "/" { + path = fmt.Sprintf("lumberjack:/%%2F%s", v[1:]) + } else { + path = fmt.Sprintf("lumberjack:/%s", v) + } + } else { + path = v + } + outputPaths = append(outputPaths, path) + } + } + + // setup log rotation + if logRotationEnable != 0 { + _ = setupLogRotation(outputs, logRotateConfigJSON) + } + config := loadConfig(level) + config.OutputPaths = outputPaths p, err := config.Build(zap.AddCallerSkip(1)) FatalIfError(err) logger = p.Sugar() } -// InitRotateLog is an initialization for a rotated logger by lumberjack -func InitRotateLog(logLevel string, ll *lumberjack.Logger) { - config := loadConfig(logLevel) - config.OutputPaths = []string{fmt.Sprintf("lumberjack:%s", ll.Filename), "stdout"} - err := zap.RegisterSink("lumberjack", func(*url.URL) (zap.Sink, error) { - return lumberjackSink{ - Logger: ll, - }, nil - }) - FatalIfError(err) +type lumberjackSink struct { + *lumberjack.Logger +} - p, err := config.Build(zap.AddCallerSkip(1)) +func (lumberjackSink) Sync() error { + return nil +} + +// setupLogRotation initializes log rotation for a single file path target. +func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { + var lumberjackSink lumberjackSink + outputFilePaths := 0 + for _, v := range logOutputs { + switch v { + case "stdout", "stderr": + continue + default: + outputFilePaths++ + } + } + // log rotation requires file target + if len(logOutputs) == 1 && outputFilePaths == 0 { + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + } + // support max 1 file target for log rotation + if outputFilePaths > 1 { + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + } + + if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil { + var unmarshalTypeError *json.UnmarshalTypeError + var syntaxError *json.SyntaxError + switch { + case errors.As(err, &syntaxError): + FatalIfError(fmt.Errorf("improperly formatted log rotation config: %w", err)) + case errors.As(err, &unmarshalTypeError): + FatalIfError(fmt.Errorf("invalid log rotation config: %w", err)) + } + } + err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) { + lumberjackSink.Filename = u.Path[1:] + return &lumberjackSink, nil + }) FatalIfError(err) - logger = p.Sugar() + return nil } func loadConfig(logLevel string) zap.Config { diff --git a/dtmcli/logger/logger_test.go b/dtmcli/logger/logger_test.go index bcba763..8516d76 100644 --- a/dtmcli/logger/logger_test.go +++ b/dtmcli/logger/logger_test.go @@ -4,19 +4,39 @@ import ( "os" "testing" - "github.com/natefinch/lumberjack" "go.uber.org/zap" ) func TestInitLog(t *testing.T) { os.Setenv("DTM_DEBUG", "1") - InitLog("debug") + InitLog("debug", nil, 0, "") Debugf("a debug msg") Infof("a info msg") Warnf("a warn msg") Errorf("a error msg") FatalfIf(false, "nothing") FatalIfError(nil) + + InitLog("debug", []string{"test.log", "stdout"}, 0, "") + Debugf("a debug msg to console and file") + Infof("a info msg to console and file") + Warnf("a warn msg to console and file") + Errorf("a error msg to console and file") + + InitLog("debug", []string{"stdout", "stderr"}, 0, "") + Debugf("a debug msg to stdout and stderr") + Infof("a info msg to stdout and stderr") + Warnf("a warn msg to stdout and stderr") + Errorf("a error msg to stdout and stderr") + + InitLog("debug", []string{"test.log", "stdout"}, 1, + "{\"maxsize\": 1, \"maxage\": 1, \"maxbackups\": 1, \"compress\": false}") + Debugf("a debug msg to console and file with rotation") + Infof("a info msg to console and file with rotation") + Warnf("a warn msg to console and file with rotation") + Errorf("a error msg to console and file with rotation") + + _ = os.Remove("test.log") } func TestWithLogger(t *testing.T) { @@ -29,22 +49,3 @@ func TestWithLogger(t *testing.T) { FatalfIf(false, "nothing") FatalIfError(nil) } - -func TestInitRotateLog(t *testing.T) { - os.Setenv("DTM_DEBUG", "1") - ll := lumberjack.Logger{ - Filename: "test.log", - MaxSize: 1, - MaxBackups: 1, - MaxAge: 1, - Compress: false, - } - InitRotateLog("debug", &ll) - Debugf("a debug msg") - Infof("a info msg") - Warnf("a warn msg") - Errorf("a error msg") - FatalfIf(false, "nothing") - FatalIfError(nil) - _ = os.Remove("test.log") -} diff --git a/dtmsvr/config/config.go b/dtmsvr/config/config.go index cc74975..848cabb 100644 --- a/dtmsvr/config/config.go +++ b/dtmsvr/config/config.go @@ -31,13 +31,10 @@ type MicroService struct { // Log config customize log type Log struct { - Level string `yaml:"Level" default:"info"` - Output string `yaml:"Output" default:"console"` - FileName string `yaml:"FileName" default:"/tmp/dtm.log"` - FileMaxSize int64 `yaml:"FileMaxSize" default:"10"` - FileMaxBackups int64 `yaml:"FileMaxBackups" default:"5"` - FileMaxAge int64 `yaml:"FileMaxAge" default:"30"` - FileCompress int64 `yaml:"FileCompress" default:"0"` + Level string `yaml:"Level" default:"info"` + Outputs string `yaml:"Outputs" default:""` + LogRotationEnable int64 `yaml:"LogRotationEnable" default:"0"` + LogRotationConfigJson string `yaml:"LogRotationConfigJson" default:""` } // Store defines storage relevant info diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index 8f25501..5a9ca89 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "net" + "strings" "time" "github.com/dtm-labs/dtm/dtmcli" @@ -20,23 +21,18 @@ import ( "github.com/dtm-labs/dtm/dtmutil" "github.com/dtm-labs/dtmdriver" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/natefinch/lumberjack" "google.golang.org/grpc" ) // StartSvr StartSvr func StartSvr() { logger.Infof("start dtmsvr") - if conf.Log.Output == "file" { - ll := lumberjack.Logger{ - Filename: conf.Log.FileName, - MaxSize: int(conf.Log.FileMaxSize), - MaxBackups: int(conf.Log.FileMaxBackups), - MaxAge: int(conf.Log.FileMaxAge), - Compress: conf.Log.FileCompress != 0, - } - logger.InitRotateLog(conf.Log.Level, &ll) + var outputs []string + if len(conf.Log.Outputs) != 0 { + outputs = strings.Split(conf.Log.Outputs, "|") } + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) + dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second) diff --git a/test/main_test.go b/test/main_test.go index 69d29cf..5f0d42f 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -29,7 +29,7 @@ func exitIf(code int) { func TestMain(m *testing.M) { config.MustLoadConfig("") - logger.InitLog("debug") + logger.InitLog("debug", nil, 0, "") dtmcli.SetCurrentDBType(busi.BusiConf.Driver) dtmsvr.TransProcessedTestChan = make(chan string, 1) dtmsvr.NowForwardDuration = 0 * time.Second From 04f2ec41e8fd5c56809b36fc54a1d23390883c76 Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 15:45:35 +0800 Subject: [PATCH 2/4] fix CI lint problem. --- bench/main.go | 2 +- dtmcli/logger/log.go | 11 ++++++----- dtmsvr/config/config.go | 2 +- dtmsvr/svr.go | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bench/main.go b/bench/main.go index 11d49b9..028e3a8 100644 --- a/bench/main.go +++ b/bench/main.go @@ -38,7 +38,7 @@ func main() { if len(conf.Log.Outputs) != 0 { outputs = strings.Split(conf.Log.Outputs, "|") } - logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) if busi.BusiConf.Driver != "" { dtmcli.SetCurrentDBType(busi.BusiConf.Driver) svr.PrepareBenchDB() diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 0ad6e9f..6ba0614 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -17,6 +17,7 @@ import ( var logger Logger +// DefaultLogOutput is the default configuration for log output. const ( DefaultLogOutput = "default" StdErrLogOutput = "stderr" @@ -78,7 +79,7 @@ func InitLog(level string, outputs []string, logRotationEnable int64, logRotateC // setup log rotation if logRotationEnable != 0 { - _ = setupLogRotation(outputs, logRotateConfigJSON) + setupLogRotation(outputs, logRotateConfigJSON) } config := loadConfig(level) @@ -97,7 +98,7 @@ func (lumberjackSink) Sync() error { } // setupLogRotation initializes log rotation for a single file path target. -func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { +func setupLogRotation(logOutputs []string, logRotateConfigJSON string) { var lumberjackSink lumberjackSink outputFilePaths := 0 for _, v := range logOutputs { @@ -110,11 +111,11 @@ func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { } // log rotation requires file target if len(logOutputs) == 1 && outputFilePaths == 0 { - FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined")) } // support max 1 file target for log rotation if outputFilePaths > 1 { - FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined")) } if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil { @@ -132,7 +133,7 @@ func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { return &lumberjackSink, nil }) FatalIfError(err) - return nil + return } func loadConfig(logLevel string) zap.Config { diff --git a/dtmsvr/config/config.go b/dtmsvr/config/config.go index 848cabb..4cc8dda 100644 --- a/dtmsvr/config/config.go +++ b/dtmsvr/config/config.go @@ -34,7 +34,7 @@ type Log struct { Level string `yaml:"Level" default:"info"` Outputs string `yaml:"Outputs" default:""` LogRotationEnable int64 `yaml:"LogRotationEnable" default:"0"` - LogRotationConfigJson string `yaml:"LogRotationConfigJson" default:""` + LogRotationConfigJSON string `yaml:"LogRotationConfigJSON" default:""` } // Store defines storage relevant info diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index 5a9ca89..0f4ee75 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -31,7 +31,7 @@ func StartSvr() { if len(conf.Log.Outputs) != 0 { outputs = strings.Split(conf.Log.Outputs, "|") } - logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { From b9ef6511bccc3cb1f02a914f72d51784e9b4beda Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 15:50:02 +0800 Subject: [PATCH 3/4] fix CI lint problem. --- dtmcli/logger/log.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 6ba0614..429cb2f 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -133,7 +133,6 @@ func setupLogRotation(logOutputs []string, logRotateConfigJSON string) { return &lumberjackSink, nil }) FatalIfError(err) - return } func loadConfig(logLevel string) zap.Config { From 57fba0975b989649f626286eb7012b55a04b457e Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 16:21:47 +0800 Subject: [PATCH 4/4] fix merge conflict problem. --- dtmsvr/svr.go | 1 + go.mod | 1 + main.go | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index ec1a6f6..0f4ee75 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -32,6 +32,7 @@ func StartSvr() { outputs = strings.Split(conf.Log.Outputs, "|") } logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) + dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second) diff --git a/go.mod b/go.mod index ef2be14..baf57ea 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/lib/pq v1.10.3 github.com/lithammer/shortuuid v2.0.3+incompatible github.com/lithammer/shortuuid/v3 v3.0.7 + github.com/natefinch/lumberjack v2.0.0+incompatible // indirect github.com/onsi/gomega v1.16.0 github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 diff --git a/main.go b/main.go index 1b088e1..43ce895 100644 --- a/main.go +++ b/main.go @@ -59,7 +59,7 @@ func main() { } config.MustLoadConfig(*confFile) if *isDebug { - config.Config.LogLevel = "debug" + config.Config.Log.Level = "debug" } if *isReset { dtmsvr.PopulateDB(false)