improve log config.

bench/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/dtm-labs/dtm/bench/svr"
 	"github.com/dtm-labs/dtm/dtmcli"
@@ -33,7 +34,11 @@ func main() {
 	}
 	logger.Infof("starting bench server")
 	config.MustLoadConfig("")
-	logger.InitLog(conf.Log.Level)
+	var outputs []string
+	if len(conf.Log.Outputs) != 0 {
+		outputs = strings.Split(conf.Log.Outputs, "|")
+	}
+	logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson)
 	if busi.BusiConf.Driver != "" {
 		dtmcli.SetCurrentDBType(busi.BusiConf.Driver)
 		svr.PrepareBenchDB()

conf.sample.yml

@@ -49,13 +49,10 @@
 # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm
 
 # Log:
-#   Level: 'info'              # default: info. can be debug|info|warn|error
-#   Output: 'console'          # default: console. can be console|file
-#   FileName: '/tmp/dtm.log'   # default: /tmp/dtm.log.
-#   FileMaxSize: 10            # default: 10, unit: MB.
-#   FileMaxBackups: 5          # default: 5.
-#   FileMaxAge: 30             # default: 30, unit: days.
-#   FileCompress: 0            # default: 0. can by 0|1, means false|true
+#   Level: 'info'                 # default: info. can be debug|info|warn|error
+#   Outputs:   ''                 # default: stdout, split by |, you can append files to Outputs if need. example:'stdout|/tmp/test.log'
+#   LogRotationEnable: 0          # default: 0
+#   LogRotationConfigJson: ''     # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}'
 
 # HttpPort: 36789
 # GrpcPort: 36790

dtmcli/logger/log.go

@@ -1,6 +1,8 @@
 package logger
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net/url"
@@ -15,8 +17,14 @@ import (
 
 var logger Logger
 
+const (
+	DefaultLogOutput = "default"
+	StdErrLogOutput  = "stderr"
+	StdOutLogOutput  = "stdout"
+)
+
 func init() {
-	InitLog(os.Getenv("LOG_LEVEL"))
+	InitLog(os.Getenv("LOG_LEVEL"), nil, 0, "")
 }
 
 // Logger logger interface
@@ -27,14 +35,6 @@ type Logger interface {
 	Errorf(format string, args ...interface{})
 }
 
-type lumberjackSink struct {
-	*lumberjack.Logger
-}
-
-func (lumberjackSink) Sync() error {
-	return nil
-}
-
 // WithLogger replaces default logger
 func WithLogger(log Logger) {
 	logger = log
@@ -42,27 +42,97 @@ func WithLogger(log Logger) {
 
 // InitLog is an initialization for a logger
 // level can be: debug info warn error
-func InitLog(level string) {
+func InitLog(level string, outputs []string, logRotationEnable int64, logRotateConfigJSON string) {
+	if len(outputs) == 0 {
+		outputs = []string{DefaultLogOutput}
+	}
+
+	// parse outputs
+	outputPaths := make([]string, 0)
+	for _, v := range outputs {
+		switch v {
+		case DefaultLogOutput:
+			outputPaths = append(outputPaths, StdOutLogOutput)
+
+		case StdErrLogOutput:
+			outputPaths = append(outputPaths, StdErrLogOutput)
+
+		case StdOutLogOutput:
+			outputPaths = append(outputPaths, StdOutLogOutput)
+
+		default:
+			var path string
+			if logRotationEnable != 0 {
+				// append rotate scheme to logs managed by lumberjack log rotation
+				if v[0:1] == "/" {
+					path = fmt.Sprintf("lumberjack:/%%2F%s", v[1:])
+				} else {
+					path = fmt.Sprintf("lumberjack:/%s", v)
+				}
+			} else {
+				path = v
+			}
+			outputPaths = append(outputPaths, path)
+		}
+	}
+
+	// setup log rotation
+	if logRotationEnable != 0 {
+		_ = setupLogRotation(outputs, logRotateConfigJSON)
+	}
+
 	config := loadConfig(level)
+	config.OutputPaths = outputPaths
 	p, err := config.Build(zap.AddCallerSkip(1))
 	FatalIfError(err)
 	logger = p.Sugar()
 }
 
-// InitRotateLog is an initialization for a rotated logger by lumberjack
-func InitRotateLog(logLevel string, ll *lumberjack.Logger) {
-	config := loadConfig(logLevel)
-	config.OutputPaths = []string{fmt.Sprintf("lumberjack:%s", ll.Filename), "stdout"}
-	err := zap.RegisterSink("lumberjack", func(*url.URL) (zap.Sink, error) {
-		return lumberjackSink{
-			Logger: ll,
-		}, nil
-	})
-	FatalIfError(err)
+type lumberjackSink struct {
+	*lumberjack.Logger
+}
 
-	p, err := config.Build(zap.AddCallerSkip(1))
+func (lumberjackSink) Sync() error {
+	return nil
+}
+
+// setupLogRotation initializes log rotation for a single file path target.
+func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error {
+	var lumberjackSink lumberjackSink
+	outputFilePaths := 0
+	for _, v := range logOutputs {
+		switch v {
+		case "stdout", "stderr":
+			continue
+		default:
+			outputFilePaths++
+		}
+	}
+	// log rotation requires file target
+	if len(logOutputs) == 1 && outputFilePaths == 0 {
+		FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined"))
+	}
+	// support max 1 file target for log rotation
+	if outputFilePaths > 1 {
+		FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJson is defined"))
+	}
+
+	if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil {
+		var unmarshalTypeError *json.UnmarshalTypeError
+		var syntaxError *json.SyntaxError
+		switch {
+		case errors.As(err, &syntaxError):
+			FatalIfError(fmt.Errorf("improperly formatted log rotation config: %w", err))
+		case errors.As(err, &unmarshalTypeError):
+			FatalIfError(fmt.Errorf("invalid log rotation config: %w", err))
+		}
+	}
+	err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) {
+		lumberjackSink.Filename = u.Path[1:]
+		return &lumberjackSink, nil
+	})
 	FatalIfError(err)
-	logger = p.Sugar()
+	return nil
 }
 
 func loadConfig(logLevel string) zap.Config {

dtmcli/logger/logger_test.go

@@ -4,19 +4,39 @@ import (
 	"os"
 	"testing"
 
-	"github.com/natefinch/lumberjack"
 	"go.uber.org/zap"
 )
 
 func TestInitLog(t *testing.T) {
 	os.Setenv("DTM_DEBUG", "1")
-	InitLog("debug")
+	InitLog("debug", nil, 0, "")
 	Debugf("a debug msg")
 	Infof("a info msg")
 	Warnf("a warn msg")
 	Errorf("a error msg")
 	FatalfIf(false, "nothing")
 	FatalIfError(nil)
+
+	InitLog("debug", []string{"test.log", "stdout"}, 0, "")
+	Debugf("a debug msg to console and file")
+	Infof("a info msg to console and file")
+	Warnf("a warn msg to console and file")
+	Errorf("a error msg to console and file")
+
+	InitLog("debug", []string{"stdout", "stderr"}, 0, "")
+	Debugf("a debug msg to stdout and stderr")
+	Infof("a info msg to stdout and stderr")
+	Warnf("a warn msg to stdout and stderr")
+	Errorf("a error msg to stdout and stderr")
+
+	InitLog("debug", []string{"test.log", "stdout"}, 1,
+		"{\"maxsize\": 1, \"maxage\": 1, \"maxbackups\": 1, \"compress\": false}")
+	Debugf("a debug msg to console and file with rotation")
+	Infof("a info msg to console and file with rotation")
+	Warnf("a warn msg to console and file with rotation")
+	Errorf("a error msg to console and file with rotation")
+
+	_ = os.Remove("test.log")
 }
 
 func TestWithLogger(t *testing.T) {
@@ -29,22 +49,3 @@ func TestWithLogger(t *testing.T) {
 	FatalfIf(false, "nothing")
 	FatalIfError(nil)
 }
-
-func TestInitRotateLog(t *testing.T) {
-	os.Setenv("DTM_DEBUG", "1")
-	ll := lumberjack.Logger{
-		Filename:   "test.log",
-		MaxSize:    1,
-		MaxBackups: 1,
-		MaxAge:     1,
-		Compress:   false,
-	}
-	InitRotateLog("debug", &ll)
-	Debugf("a debug msg")
-	Infof("a info msg")
-	Warnf("a warn msg")
-	Errorf("a error msg")
-	FatalfIf(false, "nothing")
-	FatalIfError(nil)
-	_ = os.Remove("test.log")
-}

dtmsvr/config/config.go

@@ -31,13 +31,10 @@ type MicroService struct {
 
 // Log config customize log
 type Log struct {
-	Level          string `yaml:"Level" default:"info"`
-	Output         string `yaml:"Output" default:"console"`
-	FileName       string `yaml:"FileName" default:"/tmp/dtm.log"`
-	FileMaxSize    int64  `yaml:"FileMaxSize" default:"10"`
-	FileMaxBackups int64  `yaml:"FileMaxBackups" default:"5"`
-	FileMaxAge     int64  `yaml:"FileMaxAge" default:"30"`
-	FileCompress   int64  `yaml:"FileCompress" default:"0"`
+	Level                 string `yaml:"Level" default:"info"`
+	Outputs               string `yaml:"Outputs" default:""`
+	LogRotationEnable     int64  `yaml:"LogRotationEnable" default:"0"`
+	LogRotationConfigJson string `yaml:"LogRotationConfigJson" default:""`
 }
 
 // Store defines storage relevant info

dtmsvr/svr.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 	"time"
 
 	"github.com/dtm-labs/dtm/dtmcli"
@@ -20,23 +21,18 @@ import (
 	"github.com/dtm-labs/dtm/dtmutil"
 	"github.com/dtm-labs/dtmdriver"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
-	"github.com/natefinch/lumberjack"
 	"google.golang.org/grpc"
 )
 
 // StartSvr StartSvr
 func StartSvr() {
 	logger.Infof("start dtmsvr")
-	if conf.Log.Output == "file" {
-		ll := lumberjack.Logger{
-			Filename:   conf.Log.FileName,
-			MaxSize:    int(conf.Log.FileMaxSize),
-			MaxBackups: int(conf.Log.FileMaxBackups),
-			MaxAge:     int(conf.Log.FileMaxAge),
-			Compress:   conf.Log.FileCompress != 0,
-		}
-		logger.InitRotateLog(conf.Log.Level, &ll)
+	var outputs []string
+	if len(conf.Log.Outputs) != 0 {
+		outputs = strings.Split(conf.Log.Outputs, "|")
 	}
+	logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJson)
+
 	dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second)
 	dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
 		ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second)

test/main_test.go

@@ -29,7 +29,7 @@ func exitIf(code int) {
 
 func TestMain(m *testing.M) {
 	config.MustLoadConfig("")
-	logger.InitLog("debug")
+	logger.InitLog("debug", nil, 0, "")
 	dtmcli.SetCurrentDBType(busi.BusiConf.Driver)
 	dtmsvr.TransProcessedTestChan = make(chan string, 1)
 	dtmsvr.NowForwardDuration = 0 * time.Second