Merge pull request #189 from tinattwang/main

Write log to file

bench/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/dtm-labs/dtm/bench/svr"
 	"github.com/dtm-labs/dtm/dtmcli"
@@ -33,7 +34,11 @@ func main() {
 	}
 	logger.Infof("starting bench server")
 	config.MustLoadConfig("")
-	logger.InitLog(conf.LogLevel)
+	var outputs []string
+	if len(conf.Log.Outputs) != 0 {
+		outputs = strings.Split(conf.Log.Outputs, "|")
+	}
+	logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON)
 	if busi.BusiConf.Driver != "" {
 		dtmcli.SetCurrentDBType(busi.BusiConf.Driver)
 		svr.PrepareBenchDB()

conf.sample.yml

@@ -48,7 +48,12 @@
 # RetryInterval: 10 # the subtrans branch will be retried after this interval
 # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm
 
-# LogLevel: 'info' # default: info. can be debug|info|warn|error
+# Log:
+#   Level: 'info'                 # default: info. can be debug|info|warn|error
+#   Outputs:   ''                 # default: stdout, split by |, you can append files to Outputs if need. example:'stdout|/tmp/test.log'
+#   LogRotationEnable: 0          # default: 0
+#   LogRotationConfigJson: ''     # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}'
+
 # HttpPort: 36789
 # GrpcPort: 36790
 

dtmcli/logger/log.go

@@ -1,9 +1,14 @@
 package logger
 
 import (
+	"encoding/json"
+	"errors"
+	"fmt"
 	"log"
+	"net/url"
 	"os"
 
+	"github.com/natefinch/lumberjack"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 )
@@ -12,8 +17,15 @@ import (
 
 var logger Logger
 
+// DefaultLogOutput is the default configuration for log output.
+const (
+	DefaultLogOutput = "default"
+	StdErrLogOutput  = "stderr"
+	StdOutLogOutput  = "stdout"
+)
+
 func init() {
-	InitLog(os.Getenv("LOG_LEVEL"))
+	InitLog(os.Getenv("LOG_LEVEL"), nil, 0, "")
 }
 
 // Logger logger interface
@@ -31,18 +43,108 @@ func WithLogger(log Logger) {
 
 // InitLog is an initialization for a logger
 // level can be: debug info warn error
-func InitLog(level string) {
+func InitLog(level string, outputs []string, logRotationEnable int64, logRotateConfigJSON string) {
+	if len(outputs) == 0 {
+		outputs = []string{DefaultLogOutput}
+	}
+
+	// parse outputs
+	outputPaths := make([]string, 0)
+	for _, v := range outputs {
+		switch v {
+		case DefaultLogOutput:
+			outputPaths = append(outputPaths, StdOutLogOutput)
+
+		case StdErrLogOutput:
+			outputPaths = append(outputPaths, StdErrLogOutput)
+
+		case StdOutLogOutput:
+			outputPaths = append(outputPaths, StdOutLogOutput)
+
+		default:
+			var path string
+			if logRotationEnable != 0 {
+				// append rotate scheme to logs managed by lumberjack log rotation
+				if v[0:1] == "/" {
+					path = fmt.Sprintf("lumberjack:/%%2F%s", v[1:])
+				} else {
+					path = fmt.Sprintf("lumberjack:/%s", v)
+				}
+			} else {
+				path = v
+			}
+			outputPaths = append(outputPaths, path)
+		}
+	}
+
+	// setup log rotation
+	if logRotationEnable != 0 {
+		setupLogRotation(outputs, logRotateConfigJSON)
+	}
+
+	config := loadConfig(level)
+	config.OutputPaths = outputPaths
+	p, err := config.Build(zap.AddCallerSkip(1))
+	FatalIfError(err)
+	logger = p.Sugar()
+}
+
+type lumberjackSink struct {
+	*lumberjack.Logger
+}
+
+func (lumberjackSink) Sync() error {
+	return nil
+}
+
+// setupLogRotation initializes log rotation for a single file path target.
+func setupLogRotation(logOutputs []string, logRotateConfigJSON string) {
+	var lumberjackSink lumberjackSink
+	outputFilePaths := 0
+	for _, v := range logOutputs {
+		switch v {
+		case "stdout", "stderr":
+			continue
+		default:
+			outputFilePaths++
+		}
+	}
+	// log rotation requires file target
+	if len(logOutputs) == 1 && outputFilePaths == 0 {
+		FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined"))
+	}
+	// support max 1 file target for log rotation
+	if outputFilePaths > 1 {
+		FatalIfError(fmt.Errorf("log outputs requires a single file path when LogRotationConfigJSON is defined"))
+	}
+
+	if err := json.Unmarshal([]byte(logRotateConfigJSON), &lumberjackSink); err != nil {
+		var unmarshalTypeError *json.UnmarshalTypeError
+		var syntaxError *json.SyntaxError
+		switch {
+		case errors.As(err, &syntaxError):
+			FatalIfError(fmt.Errorf("improperly formatted log rotation config: %w", err))
+		case errors.As(err, &unmarshalTypeError):
+			FatalIfError(fmt.Errorf("invalid log rotation config: %w", err))
+		}
+	}
+	err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) {
+		lumberjackSink.Filename = u.Path[1:]
+		return &lumberjackSink, nil
+	})
+	FatalIfError(err)
+}
+
+func loadConfig(logLevel string) zap.Config {
 	config := zap.NewProductionConfig()
-	err := config.Level.UnmarshalText([]byte(level))
+	err := config.Level.UnmarshalText([]byte(logLevel))
 	FatalIfError(err)
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	if os.Getenv("DTM_DEBUG") != "" {
 		config.Encoding = "console"
 		config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
 	}
-	p, err := config.Build(zap.AddCallerSkip(1))
+	return config
-	FatalIfError(err)
-	logger = p.Sugar()
 }
 
 // Debugf log to level debug

dtmcli/logger/logger_test.go

@@ -9,13 +9,34 @@ import (
 
 func TestInitLog(t *testing.T) {
 	os.Setenv("DTM_DEBUG", "1")
-	InitLog("debug")
+	InitLog("debug", nil, 0, "")
 	Debugf("a debug msg")
 	Infof("a info msg")
 	Warnf("a warn msg")
 	Errorf("a error msg")
 	FatalfIf(false, "nothing")
 	FatalIfError(nil)
+
+	InitLog("debug", []string{"test.log", "stdout"}, 0, "")
+	Debugf("a debug msg to console and file")
+	Infof("a info msg to console and file")
+	Warnf("a warn msg to console and file")
+	Errorf("a error msg to console and file")
+
+	InitLog("debug", []string{"stdout", "stderr"}, 0, "")
+	Debugf("a debug msg to stdout and stderr")
+	Infof("a info msg to stdout and stderr")
+	Warnf("a warn msg to stdout and stderr")
+	Errorf("a error msg to stdout and stderr")
+
+	InitLog("debug", []string{"test.log", "stdout"}, 1,
+		"{\"maxsize\": 1, \"maxage\": 1, \"maxbackups\": 1, \"compress\": false}")
+	Debugf("a debug msg to console and file with rotation")
+	Infof("a info msg to console and file with rotation")
+	Warnf("a warn msg to console and file with rotation")
+	Errorf("a error msg to console and file with rotation")
+
+	_ = os.Remove("test.log")
 }
 
 func TestWithLogger(t *testing.T) {

dtmsvr/config/config.go

@@ -29,6 +29,14 @@ type MicroService struct {
 	EndPoint string `yaml:"EndPoint"`
 }
 
+// Log config customize log
+type Log struct {
+	Level                 string `yaml:"Level" default:"info"`
+	Outputs               string `yaml:"Outputs" default:""`
+	LogRotationEnable     int64  `yaml:"LogRotationEnable" default:"0"`
+	LogRotationConfigJSON string `yaml:"LogRotationConfigJSON" default:""`
+}
+
 // Store defines storage relevant info
 type Store struct {
 	Driver             string `yaml:"Driver" default:"boltdb"`
@@ -72,7 +80,7 @@ type configType struct {
 	MicroService                  MicroService `yaml:"MicroService"`
 	UpdateBranchSync              int64        `yaml:"UpdateBranchSync"`
 	UpdateBranchAsyncGoroutineNum int64        `yaml:"UpdateBranchAsyncGoroutineNum" default:"1"`
-	LogLevel                      string       `yaml:"LogLevel" default:"info"`
+	Log                           Log          `yaml:"Log"`
 }
 
 // Config 配置

dtmsvr/svr.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 	"time"
 
 	"github.com/dtm-labs/dtm/dtmcli"
@@ -26,6 +27,12 @@ import (
 // StartSvr StartSvr
 func StartSvr() {
 	logger.Infof("start dtmsvr")
+	var outputs []string
+	if len(conf.Log.Outputs) != 0 {
+		outputs = strings.Split(conf.Log.Outputs, "|")
+	}
+	logger.InitLog(conf.Log.Level, outputs, conf.Log.LogRotationEnable, conf.Log.LogRotationConfigJSON)
+
 	dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second)
 	dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
 		ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second)

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/lib/pq v1.10.3
 	github.com/lithammer/shortuuid v2.0.3+incompatible
 	github.com/lithammer/shortuuid/v3 v3.0.7
+	github.com/natefinch/lumberjack v2.0.0+incompatible // indirect
 	github.com/onsi/gomega v1.16.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/stretchr/testify v1.7.0

main.go

@@ -59,7 +59,7 @@ func main() {
 	}
 	config.MustLoadConfig(*confFile)
 	if *isDebug {
-		config.Config.LogLevel = "debug"
+		config.Config.Log.Level = "debug"
 	}
 	if *isReset {
 		dtmsvr.PopulateDB(false)

test/main_test.go

@@ -29,7 +29,7 @@ func exitIf(code int) {
 
 func TestMain(m *testing.M) {
 	config.MustLoadConfig("")
-	logger.InitLog("debug")
+	logger.InitLog("debug", nil, 0, "")
 	dtmcli.SetCurrentDBType(busi.BusiConf.Driver)
 	dtmsvr.TransProcessedTestChan = make(chan string, 1)
 	dtmsvr.NowForwardDuration = 0 * time.Second