From 1dd23e3ff22217ba88e6de941f642e5c89c16030 Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 15:33:52 +0800 Subject: [PATCH 1/6] improve log config. --- bench/main.go | 7 ++- conf.sample.yml | 11 ++-- dtmcli/logger/log.go | 114 ++++++++++++++++++++++++++++------- dtmcli/logger/logger_test.go | 43 ++++++------- dtmsvr/config/config.go | 11 ++-- dtmsvr/svr.go | 16 ++--- test/main_test.go | 2 +- 7 files changed, 135 insertions(+), 69 deletions(-) diff --git a/bench/main.go b/bench/main.go index 50e488b..11d49b9 100644 --- a/bench/main.go +++ b/bench/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "strings" "github.com/dtm-labs/dtm/bench/svr" "github.com/dtm-labs/dtm/dtmcli" @@ -33,7 +34,11 @@ func main() { } logger.Infof("starting bench server") config.MustLoadConfig("") - logger.InitLog(conf.Log.Level) + var outputs []string + if len(conf.Log.Outputs) != 0 { + outputs = strings.Split(conf.Log.Outputs, "|") + } + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) if busi.BusiConf.Driver != "" { dtmcli.SetCurrentDBType(busi.BusiConf.Driver) svr.PrepareBenchDB() diff --git a/conf.sample.yml b/conf.sample.yml index 5c046ed..71d56d7 100644 --- a/conf.sample.yml +++ b/conf.sample.yml @@ -49,13 +49,10 @@ # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm # Log: -# Level: 'info' # default: info. can be debug|info|warn|error -# Output: 'console' # default: console. can be console|file -# FileName: '/tmp/dtm.log' # default: /tmp/dtm.log. -# FileMaxSize: 10 # default: 10, unit: MB. -# FileMaxBackups: 5 # default: 5. -# FileMaxAge: 30 # default: 30, unit: days. -# FileCompress: 0 # default: 0. can by 0|1, means false|true +# Level: 'info' # default: info. can be debug|info|warn|error +# Outputs: '' # default: stdout, split by |, you can append files to Outputs if need. example:'stdout|/tmp/test.log' +# LogRotationEnable: 0 # default: 0 +# LogRotationConfigJson: '' # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}' # HttpPort: 36789 # GrpcPort: 36790 diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index e2ef110..0ad6e9f 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -1,6 +1,8 @@ package logger import ( + "encoding/json" + "errors" "fmt" "log" "net/url" @@ -15,8 +17,14 @@ import ( var logger Logger +const ( + DefaultLogOutput = "default" + StdErrLogOutput = "stderr" + StdOutLogOutput = "stdout" +) + func init() { - InitLog(os.Getenv("LOG_LEVEL")) + InitLog(os.Getenv("LOG_LEVEL"), nil, 0, "") } // Logger logger interface @@ -27,14 +35,6 @@ type Logger interface { Errorf(format string, args ...interface{}) } -type lumberjackSink struct { - *lumberjack.Logger -} - -func (lumberjackSink) Sync() error { - return nil -} - // WithLogger replaces default logger func WithLogger(log Logger) { logger = log @@ -42,27 +42,97 @@ func WithLogger(log Logger) { // InitLog is an initialization for a logger // level can be: debug info warn error -func InitLog(level string) { +func InitLog(level string, outputs []string, logRotationEnable int64, logRotateConfigJSON string) { + if len(outputs) == 0 { + outputs = []string{DefaultLogOutput} + } + + // parse outputs + outputPaths := make([]string, 0) + for _, v := range outputs { + switch v { + case DefaultLogOutput: + outputPaths = append(outputPaths, StdOutLogOutput) + + case StdErrLogOutput: + outputPaths = append(outputPaths, StdErrLogOutput) + + case StdOutLogOutput: + outputPaths = append(outputPaths, StdOutLogOutput) + + default: + var path string + if logRotationEnable != 0 { + // append rotate scheme to logs managed by lumberjack log rotation + if v[0:1] == "/" { + path = fmt.Sprintf("lumberjack:/%%2F%s", v[1:]) + } else { + path = fmt.Sprintf("lumberjack:/%s", v) + } + } else { + path = v + } + outputPaths = append(outputPaths, path) + } + } + + // setup log rotation + if logRotationEnable != 0 { + _ = setupLogRotation(outputs, logRotateConfigJSON) + } + config := loadConfig(level) + config.OutputPaths = outputPaths p, err := config.Build(zap.AddCallerSkip(1)) FatalIfError(err) logger = p.Sugar() } -// InitRotateLog is an initialization for a rotated logger by lumberjack -func InitRotateLog(logLevel string, ll *lumberjack.Logger) { - config := loadConfig(logLevel) - config.OutputPaths = []string{fmt.Sprintf("lumberjack:%s", ll.Filename), "stdout"} - err := zap.RegisterSink("lumberjack", func(*url.URL) (zap.Sink, error) { - return lumberjackSink{ - Logger: ll, - }, nil - }) - FatalIfError(err) +type lumberjackSink struct { + *lumberjack.Logger +} - p, err := config.Build(zap.AddCallerSkip(1)) +func (lumberjackSink) Sync() error { + return nil +} + +// setupLogRotation initializes log rotation for a single file path target. +func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { + var lumberjackSink lumberjackSink + outputFilePaths := 0 + for _, v := range logOutputs { + switch v { + case "stdout", "stderr": + continue + default: + outputFilePaths++ + } + } + // log rotation requires file target + if len(logOutputs) == 1 && outputFilePaths == 0 { + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + } + // support max 1 file target for log rotation + if outputFilePaths > 1 { + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + } + + if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil { + var unmarshalTypeError *json.UnmarshalTypeError + var syntaxError *json.SyntaxError + switch { + case errors.As(err, &syntaxError): + FatalIfError(fmt.Errorf("improperly formatted log rotation config: %w", err)) + case errors.As(err, &unmarshalTypeError): + FatalIfError(fmt.Errorf("invalid log rotation config: %w", err)) + } + } + err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) { + lumberjackSink.Filename = u.Path[1:] + return &lumberjackSink, nil + }) FatalIfError(err) - logger = p.Sugar() + return nil } func loadConfig(logLevel string) zap.Config { diff --git a/dtmcli/logger/logger_test.go b/dtmcli/logger/logger_test.go index bcba763..8516d76 100644 --- a/dtmcli/logger/logger_test.go +++ b/dtmcli/logger/logger_test.go @@ -4,19 +4,39 @@ import ( "os" "testing" - "github.com/natefinch/lumberjack" "go.uber.org/zap" ) func TestInitLog(t *testing.T) { os.Setenv("DTM_DEBUG", "1") - InitLog("debug") + InitLog("debug", nil, 0, "") Debugf("a debug msg") Infof("a info msg") Warnf("a warn msg") Errorf("a error msg") FatalfIf(false, "nothing") FatalIfError(nil) + + InitLog("debug", []string{"test.log", "stdout"}, 0, "") + Debugf("a debug msg to console and file") + Infof("a info msg to console and file") + Warnf("a warn msg to console and file") + Errorf("a error msg to console and file") + + InitLog("debug", []string{"stdout", "stderr"}, 0, "") + Debugf("a debug msg to stdout and stderr") + Infof("a info msg to stdout and stderr") + Warnf("a warn msg to stdout and stderr") + Errorf("a error msg to stdout and stderr") + + InitLog("debug", []string{"test.log", "stdout"}, 1, + "{\"maxsize\": 1, \"maxage\": 1, \"maxbackups\": 1, \"compress\": false}") + Debugf("a debug msg to console and file with rotation") + Infof("a info msg to console and file with rotation") + Warnf("a warn msg to console and file with rotation") + Errorf("a error msg to console and file with rotation") + + _ = os.Remove("test.log") } func TestWithLogger(t *testing.T) { @@ -29,22 +49,3 @@ func TestWithLogger(t *testing.T) { FatalfIf(false, "nothing") FatalIfError(nil) } - -func TestInitRotateLog(t *testing.T) { - os.Setenv("DTM_DEBUG", "1") - ll := lumberjack.Logger{ - Filename: "test.log", - MaxSize: 1, - MaxBackups: 1, - MaxAge: 1, - Compress: false, - } - InitRotateLog("debug", &ll) - Debugf("a debug msg") - Infof("a info msg") - Warnf("a warn msg") - Errorf("a error msg") - FatalfIf(false, "nothing") - FatalIfError(nil) - _ = os.Remove("test.log") -} diff --git a/dtmsvr/config/config.go b/dtmsvr/config/config.go index cc74975..848cabb 100644 --- a/dtmsvr/config/config.go +++ b/dtmsvr/config/config.go @@ -31,13 +31,10 @@ type MicroService struct { // Log config customize log type Log struct { - Level string `yaml:"Level" default:"info"` - Output string `yaml:"Output" default:"console"` - FileName string `yaml:"FileName" default:"/tmp/dtm.log"` - FileMaxSize int64 `yaml:"FileMaxSize" default:"10"` - FileMaxBackups int64 `yaml:"FileMaxBackups" default:"5"` - FileMaxAge int64 `yaml:"FileMaxAge" default:"30"` - FileCompress int64 `yaml:"FileCompress" default:"0"` + Level string `yaml:"Level" default:"info"` + Outputs string `yaml:"Outputs" default:""` + LogRotationEnable int64 `yaml:"LogRotationEnable" default:"0"` + LogRotationConfigJson string `yaml:"LogRotationConfigJson" default:""` } // Store defines storage relevant info diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index 8f25501..5a9ca89 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "net" + "strings" "time" "github.com/dtm-labs/dtm/dtmcli" @@ -20,23 +21,18 @@ import ( "github.com/dtm-labs/dtm/dtmutil" "github.com/dtm-labs/dtmdriver" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/natefinch/lumberjack" "google.golang.org/grpc" ) // StartSvr StartSvr func StartSvr() { logger.Infof("start dtmsvr") - if conf.Log.Output == "file" { - ll := lumberjack.Logger{ - Filename: conf.Log.FileName, - MaxSize: int(conf.Log.FileMaxSize), - MaxBackups: int(conf.Log.FileMaxBackups), - MaxAge: int(conf.Log.FileMaxAge), - Compress: conf.Log.FileCompress != 0, - } - logger.InitRotateLog(conf.Log.Level, &ll) + var outputs []string + if len(conf.Log.Outputs) != 0 { + outputs = strings.Split(conf.Log.Outputs, "|") } + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) + dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second) diff --git a/test/main_test.go b/test/main_test.go index 69d29cf..5f0d42f 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -29,7 +29,7 @@ func exitIf(code int) { func TestMain(m *testing.M) { config.MustLoadConfig("") - logger.InitLog("debug") + logger.InitLog("debug", nil, 0, "") dtmcli.SetCurrentDBType(busi.BusiConf.Driver) dtmsvr.TransProcessedTestChan = make(chan string, 1) dtmsvr.NowForwardDuration = 0 * time.Second From 04f2ec41e8fd5c56809b36fc54a1d23390883c76 Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 15:45:35 +0800 Subject: [PATCH 2/6] fix CI lint problem. --- bench/main.go | 2 +- dtmcli/logger/log.go | 11 ++++++----- dtmsvr/config/config.go | 2 +- dtmsvr/svr.go | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bench/main.go b/bench/main.go index 11d49b9..028e3a8 100644 --- a/bench/main.go +++ b/bench/main.go @@ -38,7 +38,7 @@ func main() { if len(conf.Log.Outputs) != 0 { outputs = strings.Split(conf.Log.Outputs, "|") } - logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) if busi.BusiConf.Driver != "" { dtmcli.SetCurrentDBType(busi.BusiConf.Driver) svr.PrepareBenchDB() diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 0ad6e9f..6ba0614 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -17,6 +17,7 @@ import ( var logger Logger +// DefaultLogOutput is the default configuration for log output. const ( DefaultLogOutput = "default" StdErrLogOutput = "stderr" @@ -78,7 +79,7 @@ func InitLog(level string, outputs []string, logRotationEnable int64, logRotateC // setup log rotation if logRotationEnable != 0 { - _ = setupLogRotation(outputs, logRotateConfigJSON) + setupLogRotation(outputs, logRotateConfigJSON) } config := loadConfig(level) @@ -97,7 +98,7 @@ func (lumberjackSink) Sync() error { } // setupLogRotation initializes log rotation for a single file path target. -func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { +func setupLogRotation(logOutputs []string, logRotateConfigJSON string) { var lumberjackSink lumberjackSink outputFilePaths := 0 for _, v := range logOutputs { @@ -110,11 +111,11 @@ func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { } // log rotation requires file target if len(logOutputs) == 1 && outputFilePaths == 0 { - FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined")) } // support max 1 file target for log rotation if outputFilePaths > 1 { - FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined")) + FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined")) } if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil { @@ -132,7 +133,7 @@ func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { return &lumberjackSink, nil }) FatalIfError(err) - return nil + return } func loadConfig(logLevel string) zap.Config { diff --git a/dtmsvr/config/config.go b/dtmsvr/config/config.go index 848cabb..4cc8dda 100644 --- a/dtmsvr/config/config.go +++ b/dtmsvr/config/config.go @@ -34,7 +34,7 @@ type Log struct { Level string `yaml:"Level" default:"info"` Outputs string `yaml:"Outputs" default:""` LogRotationEnable int64 `yaml:"LogRotationEnable" default:"0"` - LogRotationConfigJson string `yaml:"LogRotationConfigJson" default:""` + LogRotationConfigJSON string `yaml:"LogRotationConfigJSON" default:""` } // Store defines storage relevant info diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index 5a9ca89..0f4ee75 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -31,7 +31,7 @@ func StartSvr() { if len(conf.Log.Outputs) != 0 { outputs = strings.Split(conf.Log.Outputs, "|") } - logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson) + logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { From b9ef6511bccc3cb1f02a914f72d51784e9b4beda Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 15:50:02 +0800 Subject: [PATCH 3/6] fix CI lint problem. --- dtmcli/logger/log.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 6ba0614..429cb2f 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -133,7 +133,6 @@ func setupLogRotation(logOutputs []string, logRotateConfigJSON string) { return &lumberjackSink, nil }) FatalIfError(err) - return } func loadConfig(logLevel string) zap.Config { From 57fba0975b989649f626286eb7012b55a04b457e Mon Sep 17 00:00:00 2001 From: tinatwang Date: Mon, 17 Jan 2022 16:21:47 +0800 Subject: [PATCH 4/6] fix merge conflict problem. --- dtmsvr/svr.go | 1 + go.mod | 1 + main.go | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index ec1a6f6..0f4ee75 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -32,6 +32,7 @@ func StartSvr() { outputs = strings.Split(conf.Log.Outputs, "|") } logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) + dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second) diff --git a/go.mod b/go.mod index ef2be14..baf57ea 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/lib/pq v1.10.3 github.com/lithammer/shortuuid v2.0.3+incompatible github.com/lithammer/shortuuid/v3 v3.0.7 + github.com/natefinch/lumberjack v2.0.0+incompatible // indirect github.com/onsi/gomega v1.16.0 github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 diff --git a/main.go b/main.go index 1b088e1..43ce895 100644 --- a/main.go +++ b/main.go @@ -59,7 +59,7 @@ func main() { } config.MustLoadConfig(*confFile) if *isDebug { - config.Config.LogLevel = "debug" + config.Config.Log.Level = "debug" } if *isReset { dtmsvr.PopulateDB(false) From fe674db7c6caf1909cf09c7b9cf92142bb1e205e Mon Sep 17 00:00:00 2001 From: yedf2 <120050102@qq.com> Date: Mon, 17 Jan 2022 20:09:19 +0800 Subject: [PATCH 5/6] refactor log --- bench/main.go | 7 +-- conf.sample.yml | 8 +-- dtmcli/logger/log.go | 94 +++++++++--------------------------- dtmcli/logger/logger_test.go | 22 ++------- dtmsvr/config/config.go | 8 +-- dtmsvr/svr.go | 7 --- main.go | 4 +- test/main_test.go | 2 +- 8 files changed, 41 insertions(+), 111 deletions(-) diff --git a/bench/main.go b/bench/main.go index 028e3a8..34bd859 100644 --- a/bench/main.go +++ b/bench/main.go @@ -3,7 +3,6 @@ package main import ( "fmt" "os" - "strings" "github.com/dtm-labs/dtm/bench/svr" "github.com/dtm-labs/dtm/dtmcli" @@ -34,11 +33,7 @@ func main() { } logger.Infof("starting bench server") config.MustLoadConfig("") - var outputs []string - if len(conf.Log.Outputs) != 0 { - outputs = strings.Split(conf.Log.Outputs, "|") - } - logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) + logger.InitLog(conf.LogLevel) if busi.BusiConf.Driver != "" { dtmcli.SetCurrentDBType(busi.BusiConf.Driver) svr.PrepareBenchDB() diff --git a/conf.sample.yml b/conf.sample.yml index 71d56d7..9dabfc3 100644 --- a/conf.sample.yml +++ b/conf.sample.yml @@ -48,11 +48,11 @@ # RetryInterval: 10 # the subtrans branch will be retried after this interval # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm +# LogLevel: 'info' # default: info. can be debug|info|warn|error # Log: -# Level: 'info' # default: info. can be debug|info|warn|error -# Outputs: '' # default: stdout, split by |, you can append files to Outputs if need. example:'stdout|/tmp/test.log' -# LogRotationEnable: 0 # default: 0 -# LogRotationConfigJson: '' # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}' +# Outputs: 'stderr' # default: stderr, split by |, you can append files to Outputs if need. example:'stderr|/tmp/test.log' +# RotationEnable: 0 # default: 0 +# RotationConfigJson: '{}' # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}' # HttpPort: 36789 # GrpcPort: 36790 diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 429cb2f..9f6412d 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -2,11 +2,11 @@ package logger import ( "encoding/json" - "errors" "fmt" "log" "net/url" "os" + "strings" "github.com/natefinch/lumberjack" "go.uber.org/zap" @@ -17,15 +17,15 @@ import ( var logger Logger -// DefaultLogOutput is the default configuration for log output. const ( - DefaultLogOutput = "default" - StdErrLogOutput = "stderr" - StdOutLogOutput = "stdout" + // StdErr is the default configuration for log output. + StdErr = "stderr" + // StdOut configuration for log output + StdOut = "stdout" ) func init() { - InitLog(os.Getenv("LOG_LEVEL"), nil, 0, "") + InitLog(os.Getenv("LOG_LEVEL")) } // Logger logger interface @@ -43,43 +43,21 @@ func WithLogger(log Logger) { // InitLog is an initialization for a logger // level can be: debug info warn error -func InitLog(level string, outputs []string, logRotationEnable int64, logRotateConfigJSON string) { - if len(outputs) == 0 { - outputs = []string{DefaultLogOutput} - } +func InitLog(level string) { + InitLog2(level, StdOut, 0, "") +} - // parse outputs - outputPaths := make([]string, 0) - for _, v := range outputs { - switch v { - case DefaultLogOutput: - outputPaths = append(outputPaths, StdOutLogOutput) - - case StdErrLogOutput: - outputPaths = append(outputPaths, StdErrLogOutput) - - case StdOutLogOutput: - outputPaths = append(outputPaths, StdOutLogOutput) - - default: - var path string - if logRotationEnable != 0 { - // append rotate scheme to logs managed by lumberjack log rotation - if v[0:1] == "/" { - path = fmt.Sprintf("lumberjack:/%%2F%s", v[1:]) - } else { - path = fmt.Sprintf("lumberjack:/%s", v) - } - } else { - path = v - } - outputPaths = append(outputPaths, path) +// InitLog2 specify advanced log config +func InitLog2(level string, outputs string, logRotationEnable int64, logRotateConfigJSON string) { + outputPaths := strings.Split(outputs, "|") + for i, v := range outputPaths { + if logRotationEnable != 0 && v != StdErr && v != StdOut { + outputPaths[i] = fmt.Sprintf("lumberjack://%s", v) } } - // setup log rotation if logRotationEnable != 0 { - setupLogRotation(outputs, logRotateConfigJSON) + setupLogRotation(outputPaths, logRotateConfigJSON) } config := loadConfig(level) @@ -90,47 +68,21 @@ func InitLog(level string, outputs []string, logRotationEnable int64, logRotateC } type lumberjackSink struct { - *lumberjack.Logger + lumberjack.Logger } -func (lumberjackSink) Sync() error { +func (*lumberjackSink) Sync() error { return nil } // setupLogRotation initializes log rotation for a single file path target. func setupLogRotation(logOutputs []string, logRotateConfigJSON string) { - var lumberjackSink lumberjackSink - outputFilePaths := 0 - for _, v := range logOutputs { - switch v { - case "stdout", "stderr": - continue - default: - outputFilePaths++ - } - } - // log rotation requires file target - if len(logOutputs) == 1 && outputFilePaths == 0 { - FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined")) - } - // support max 1 file target for log rotation - if outputFilePaths > 1 { - FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined")) - } - - if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil { - var unmarshalTypeError *json.UnmarshalTypeError - var syntaxError *json.SyntaxError - switch { - case errors.As(err, &syntaxError): - FatalIfError(fmt.Errorf("improperly formatted log rotation config: %w", err)) - case errors.As(err, &unmarshalTypeError): - FatalIfError(fmt.Errorf("invalid log rotation config: %w", err)) - } - } err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) { - lumberjackSink.Filename = u.Path[1:] - return &lumberjackSink, nil + var conf lumberjackSink + err := json.Unmarshal([]byte(logRotateConfigJSON), &conf) + FatalfIf(err != nil, "bad config LogRotateConfigJSON: %v", err) + conf.Filename = u.Host + u.Path + return &conf, nil }) FatalIfError(err) } diff --git a/dtmcli/logger/logger_test.go b/dtmcli/logger/logger_test.go index 8516d76..49035c2 100644 --- a/dtmcli/logger/logger_test.go +++ b/dtmcli/logger/logger_test.go @@ -9,7 +9,7 @@ import ( func TestInitLog(t *testing.T) { os.Setenv("DTM_DEBUG", "1") - InitLog("debug", nil, 0, "") + InitLog("debug") Debugf("a debug msg") Infof("a info msg") Warnf("a warn msg") @@ -17,26 +17,14 @@ func TestInitLog(t *testing.T) { FatalfIf(false, "nothing") FatalIfError(nil) - InitLog("debug", []string{"test.log", "stdout"}, 0, "") + InitLog2("debug", "test.log|stderr", 0, "") Debugf("a debug msg to console and file") - Infof("a info msg to console and file") - Warnf("a warn msg to console and file") - Errorf("a error msg to console and file") - InitLog("debug", []string{"stdout", "stderr"}, 0, "") - Debugf("a debug msg to stdout and stderr") - Infof("a info msg to stdout and stderr") - Warnf("a warn msg to stdout and stderr") - Errorf("a error msg to stdout and stderr") - - InitLog("debug", []string{"test.log", "stdout"}, 1, + InitLog2("debug", "test2.log|/tmp/dtm-test1.log|/tmp/dtm-test.log|stdout|stderr", 1, "{\"maxsize\": 1, \"maxage\": 1, \"maxbackups\": 1, \"compress\": false}") - Debugf("a debug msg to console and file with rotation") - Infof("a info msg to console and file with rotation") - Warnf("a warn msg to console and file with rotation") - Errorf("a error msg to console and file with rotation") + Debugf("a debug msg to /tmp/dtm-test.log|test2.log|stdout|stderr") - _ = os.Remove("test.log") + // _ = os.Remove("test.log") } func TestWithLogger(t *testing.T) { diff --git a/dtmsvr/config/config.go b/dtmsvr/config/config.go index 4cc8dda..06a8fda 100644 --- a/dtmsvr/config/config.go +++ b/dtmsvr/config/config.go @@ -31,10 +31,9 @@ type MicroService struct { // Log config customize log type Log struct { - Level string `yaml:"Level" default:"info"` - Outputs string `yaml:"Outputs" default:""` - LogRotationEnable int64 `yaml:"LogRotationEnable" default:"0"` - LogRotationConfigJSON string `yaml:"LogRotationConfigJSON" default:""` + Outputs string `yaml:"Outputs" default:"stderr"` + RotationEnable int64 `yaml:"RotationEnable" default:"0"` + RotationConfigJSON string `yaml:"RotationConfigJSON" default:"{}"` } // Store defines storage relevant info @@ -80,6 +79,7 @@ type configType struct { MicroService MicroService `yaml:"MicroService"` UpdateBranchSync int64 `yaml:"UpdateBranchSync"` UpdateBranchAsyncGoroutineNum int64 `yaml:"UpdateBranchAsyncGoroutineNum" default:"1"` + LogLevel string `yaml:"LogLevel" default:"info"` Log Log `yaml:"Log"` } diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index 0f4ee75..56f288c 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -10,7 +10,6 @@ import ( "context" "fmt" "net" - "strings" "time" "github.com/dtm-labs/dtm/dtmcli" @@ -27,12 +26,6 @@ import ( // StartSvr StartSvr func StartSvr() { logger.Infof("start dtmsvr") - var outputs []string - if len(conf.Log.Outputs) != 0 { - outputs = strings.Split(conf.Log.Outputs, "|") - } - logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON) - dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second) diff --git a/main.go b/main.go index 43ce895..e7b81b3 100644 --- a/main.go +++ b/main.go @@ -58,9 +58,11 @@ func main() { return } config.MustLoadConfig(*confFile) + conf := config.Config if *isDebug { - config.Config.Log.Level = "debug" + conf.LogLevel = "debug" } + logger.InitLog2(conf.LogLevel, conf.Log.Outputs, conf.Log.RotationEnable, conf.Log.RotationConfigJSON) if *isReset { dtmsvr.PopulateDB(false) } diff --git a/test/main_test.go b/test/main_test.go index 5f0d42f..69d29cf 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -29,7 +29,7 @@ func exitIf(code int) { func TestMain(m *testing.M) { config.MustLoadConfig("") - logger.InitLog("debug", nil, 0, "") + logger.InitLog("debug") dtmcli.SetCurrentDBType(busi.BusiConf.Driver) dtmsvr.TransProcessedTestChan = make(chan string, 1) dtmsvr.NowForwardDuration = 0 * time.Second From 5e491a9ced1527d72c141db6fa2435bdcaaa08f6 Mon Sep 17 00:00:00 2001 From: yedf2 <120050102@qq.com> Date: Mon, 17 Jan 2022 20:17:44 +0800 Subject: [PATCH 6/6] fix go-lint --- dtmcli/logger/log.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 9f6412d..45048b1 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -57,7 +57,7 @@ func InitLog2(level string, outputs string, logRotationEnable int64, logRotateCo } if logRotationEnable != 0 { - setupLogRotation(outputPaths, logRotateConfigJSON) + setupLogRotation(logRotateConfigJSON) } config := loadConfig(level) @@ -76,7 +76,7 @@ func (*lumberjackSink) Sync() error { } // setupLogRotation initializes log rotation for a single file path target. -func setupLogRotation(logOutputs []string, logRotateConfigJSON string) { +func setupLogRotation(logRotateConfigJSON string) { err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) { var conf lumberjackSink err := json.Unmarshal([]byte(logRotateConfigJSON), &conf)