refactor log

bench/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"strings"
 
 	"github.com/dtm-labs/dtm/bench/svr"
 	"github.com/dtm-labs/dtm/dtmcli"
@@ -34,11 +33,7 @@ func main() {
 	}
 	logger.Infof("starting bench server")
 	config.MustLoadConfig("")
-	var outputs []string
+	logger.InitLog(conf.LogLevel)
-	if len(conf.Log.Outputs) != 0 {
-		outputs = strings.Split(conf.Log.Outputs, "|")
-	}
-	logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON)
 	if busi.BusiConf.Driver != "" {
 		dtmcli.SetCurrentDBType(busi.BusiConf.Driver)
 		svr.PrepareBenchDB()

conf.sample.yml

@@ -48,11 +48,11 @@
 # RetryInterval: 10 # the subtrans branch will be retried after this interval
 # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm
 
+# LogLevel: 'info'              # default: info. can be debug|info|warn|error
 # Log:
-#   Level: 'info'                 # default: info. can be debug|info|warn|error
+#   Outputs: 'stderr'           # default: stderr, split by |, you can append files to Outputs if need. example:'stderr|/tmp/test.log'
-#   Outputs:   ''                 # default: stdout, split by |, you can append files to Outputs if need. example:'stdout|/tmp/test.log'
+#   RotationEnable: 0           # default: 0
-#   LogRotationEnable: 0          # default: 0
+#   RotationConfigJson: '{}'    # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}'
-#   LogRotationConfigJson: ''     # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}'
 
 # HttpPort: 36789
 # GrpcPort: 36790

dtmcli/logger/log.go

@@ -2,11 +2,11 @@ package logger
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"net/url"
 	"os"
+	"strings"
 
 	"github.com/natefinch/lumberjack"
 	"go.uber.org/zap"
@@ -17,15 +17,15 @@ import (
 
 var logger Logger
 
-// DefaultLogOutput is the default configuration for log output.
 const (
-	DefaultLogOutput = "default"
+	// StdErr is the default configuration for log output.
-	StdErrLogOutput  = "stderr"
+	StdErr = "stderr"
-	StdOutLogOutput  = "stdout"
+	// StdOut configuration for log output
+	StdOut = "stdout"
 )
 
 func init() {
-	InitLog(os.Getenv("LOG_LEVEL"), nil, 0, "")
+	InitLog(os.Getenv("LOG_LEVEL"))
 }
 
 // Logger logger interface
@@ -43,43 +43,21 @@ func WithLogger(log Logger) {
 
 // InitLog is an initialization for a logger
 // level can be: debug info warn error
-func InitLog(level string, outputs []string, logRotationEnable int64, logRotateConfigJSON string) {
+func InitLog(level string) {
-	if len(outputs) == 0 {
+	InitLog2(level, StdOut, 0, "")
-		outputs = []string{DefaultLogOutput}
+}
-	}
 
-	// parse outputs
+// InitLog2 specify advanced log config
-	outputPaths := make([]string, 0)
+func InitLog2(level string, outputs string, logRotationEnable int64, logRotateConfigJSON string) {
-	for _, v := range outputs {
+	outputPaths := strings.Split(outputs, "|")
-		switch v {
+	for i, v := range outputPaths {
-		case DefaultLogOutput:
+		if logRotationEnable != 0 && v != StdErr && v != StdOut {
-			outputPaths = append(outputPaths, StdOutLogOutput)
+			outputPaths[i] = fmt.Sprintf("lumberjack://%s", v)
-
-		case StdErrLogOutput:
-			outputPaths = append(outputPaths, StdErrLogOutput)
-
-		case StdOutLogOutput:
-			outputPaths = append(outputPaths, StdOutLogOutput)
-
-		default:
-			var path string
-			if logRotationEnable != 0 {
-				// append rotate scheme to logs managed by lumberjack log rotation
-				if v[0:1] == "/" {
-					path = fmt.Sprintf("lumberjack:/%%2F%s", v[1:])
-				} else {
-					path = fmt.Sprintf("lumberjack:/%s", v)
-				}
-			} else {
-				path = v
-			}
-			outputPaths = append(outputPaths, path)
 		}
 	}
 
-	// setup log rotation
 	if logRotationEnable != 0 {
-		setupLogRotation(outputs, logRotateConfigJSON)
+		setupLogRotation(outputPaths, logRotateConfigJSON)
 	}
 
 	config := loadConfig(level)
@@ -90,47 +68,21 @@ func InitLog(level string, outputs []string, logRotationEnable int64, logRotateC
 }
 
 type lumberjackSink struct {
-	*lumberjack.Logger
+	lumberjack.Logger
 }
 
-func (lumberjackSink) Sync() error {
+func (*lumberjackSink) Sync() error {
 	return nil
 }
 
 // setupLogRotation initializes log rotation for a single file path target.
 func setupLogRotation(logOutputs []string, logRotateConfigJSON string) {
-	var lumberjackSink lumberjackSink
-	outputFilePaths := 0
-	for _, v := range logOutputs {
-		switch v {
-		case "stdout", "stderr":
-			continue
-		default:
-			outputFilePaths++
-		}
-	}
-	// log rotation requires file target
-	if len(logOutputs) == 1 && outputFilePaths == 0 {
-		FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined"))
-	}
-	// support max 1 file target for log rotation
-	if outputFilePaths > 1 {
-		FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined"))
-	}
-
-	if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil {
-		var unmarshalTypeError *json.UnmarshalTypeError
-		var syntaxError *json.SyntaxError
-		switch {
-		case errors.As(err, &syntaxError):
-			FatalIfError(fmt.Errorf("improperly formatted log rotation config: %w", err))
-		case errors.As(err, &unmarshalTypeError):
-			FatalIfError(fmt.Errorf("invalid log rotation config: %w", err))
-		}
-	}
 	err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) {
-		lumberjackSink.Filename = u.Path[1:]
+		var conf lumberjackSink
-		return &lumberjackSink, nil
+		err := json.Unmarshal([]byte(logRotateConfigJSON), &conf)
+		FatalfIf(err != nil, "bad config LogRotateConfigJSON: %v", err)
+		conf.Filename = u.Host + u.Path
+		return &conf, nil
 	})
 	FatalIfError(err)
 }

dtmcli/logger/logger_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestInitLog(t *testing.T) {
 	os.Setenv("DTM_DEBUG", "1")
-	InitLog("debug", nil, 0, "")
+	InitLog("debug")
 	Debugf("a debug msg")
 	Infof("a info msg")
 	Warnf("a warn msg")
@@ -17,26 +17,14 @@ func TestInitLog(t *testing.T) {
 	FatalfIf(false, "nothing")
 	FatalIfError(nil)
 
-	InitLog("debug", []string{"test.log", "stdout"}, 0, "")
+	InitLog2("debug", "test.log|stderr", 0, "")
 	Debugf("a debug msg to console and file")
-	Infof("a info msg to console and file")
-	Warnf("a warn msg to console and file")
-	Errorf("a error msg to console and file")
 
-	InitLog("debug", []string{"stdout", "stderr"}, 0, "")
+	InitLog2("debug", "test2.log|/tmp/dtm-test1.log|/tmp/dtm-test.log|stdout|stderr", 1,
-	Debugf("a debug msg to stdout and stderr")
-	Infof("a info msg to stdout and stderr")
-	Warnf("a warn msg to stdout and stderr")
-	Errorf("a error msg to stdout and stderr")
-
-	InitLog("debug", []string{"test.log", "stdout"}, 1,
 		"{\"maxsize\": 1, \"maxage\": 1, \"maxbackups\": 1, \"compress\": false}")
-	Debugf("a debug msg to console and file with rotation")
+	Debugf("a debug msg to /tmp/dtm-test.log|test2.log|stdout|stderr")
-	Infof("a info msg to console and file with rotation")
-	Warnf("a warn msg to console and file with rotation")
-	Errorf("a error msg to console and file with rotation")
 
-	_ = os.Remove("test.log")
+	// _ = os.Remove("test.log")
 }
 
 func TestWithLogger(t *testing.T) {

dtmsvr/config/config.go

@@ -31,10 +31,9 @@ type MicroService struct {
 
 // Log config customize log
 type Log struct {
-	Level                 string `yaml:"Level" default:"info"`
+	Outputs            string `yaml:"Outputs" default:"stderr"`
-	Outputs               string `yaml:"Outputs" default:""`
+	RotationEnable     int64  `yaml:"RotationEnable" default:"0"`
-	LogRotationEnable     int64  `yaml:"LogRotationEnable" default:"0"`
+	RotationConfigJSON string `yaml:"RotationConfigJSON" default:"{}"`
-	LogRotationConfigJSON string `yaml:"LogRotationConfigJSON" default:""`
 }
 
 // Store defines storage relevant info
@@ -80,6 +79,7 @@ type configType struct {
 	MicroService                  MicroService `yaml:"MicroService"`
 	UpdateBranchSync              int64        `yaml:"UpdateBranchSync"`
 	UpdateBranchAsyncGoroutineNum int64        `yaml:"UpdateBranchAsyncGoroutineNum" default:"1"`
+	LogLevel                      string       `yaml:"LogLevel" default:"info"`
 	Log                           Log          `yaml:"Log"`
 }
 

dtmsvr/svr.go

@@ -10,7 +10,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"strings"
 	"time"
 
 	"github.com/dtm-labs/dtm/dtmcli"
@@ -27,12 +26,6 @@ import (
 // StartSvr StartSvr
 func StartSvr() {
 	logger.Infof("start dtmsvr")
-	var outputs []string
-	if len(conf.Log.Outputs) != 0 {
-		outputs = strings.Split(conf.Log.Outputs, "|")
-	}
-	logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON)
-
 	dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second)
 	dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
 		ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second)

main.go

@@ -58,9 +58,11 @@ func main() {
 		return
 	}
 	config.MustLoadConfig(*confFile)
+	conf := config.Config
 	if *isDebug {
-		config.Config.Log.Level = "debug"
+		conf.LogLevel = "debug"
 	}
+	logger.InitLog2(conf.LogLevel, conf.Log.Outputs, conf.Log.RotationEnable, conf.Log.RotationConfigJSON)
 	if *isReset {
 		dtmsvr.PopulateDB(false)
 	}

test/main_test.go

@@ -29,7 +29,7 @@ func exitIf(code int) {
 
 func TestMain(m *testing.M) {
 	config.MustLoadConfig("")
-	logger.InitLog("debug", nil, 0, "")
+	logger.InitLog("debug")
 	dtmcli.SetCurrentDBType(busi.BusiConf.Driver)
 	dtmsvr.TransProcessedTestChan = make(chan string, 1)
 	dtmsvr.NowForwardDuration = 0 * time.Second