Merge branch 'main' into main

conf.sample.yml

@@ -48,7 +48,12 @@
 # RetryInterval: 10 # the subtrans branch will be retried after this interval
 # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm
 
-# LogLevel: 'info' # default: info. can be debug|info|warn|error
+# LogLevel: 'info'              # default: info. can be debug|info|warn|error
+# Log:
+#   Outputs: 'stderr'           # default: stderr, split by |, you can append files to Outputs if need. example:'stderr|/tmp/test.log'
+#   RotationEnable: 0           # default: 0
+#   RotationConfigJson: '{}'    # example: '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}'
+
 # HttpPort: 36789
 # GrpcPort: 36790
 

dtmcli/logger/log.go

@@ -1,9 +1,14 @@
 package logger
 
 import (
+	"encoding/json"
+	"fmt"
 	"log"
+	"net/url"
 	"os"
+	"strings"
 
+	"github.com/natefinch/lumberjack"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 )
@@ -12,6 +17,13 @@ import (
 
 var logger Logger
 
+const (
+	// StdErr is the default configuration for log output.
+	StdErr = "stderr"
+	// StdOut configuration for log output
+	StdOut = "stdout"
+)
+
 func init() {
 	InitLog(os.Getenv("LOG_LEVEL"))
 }
@@ -32,17 +44,59 @@ func WithLogger(log Logger) {
 // InitLog is an initialization for a logger
 // level can be: debug info warn error
 func InitLog(level string) {
+	InitLog2(level, StdOut, 0, "")
+}
+
+// InitLog2 specify advanced log config
+func InitLog2(level string, outputs string, logRotationEnable int64, logRotateConfigJSON string) {
+	outputPaths := strings.Split(outputs, "|")
+	for i, v := range outputPaths {
+		if logRotationEnable != 0 && v != StdErr && v != StdOut {
+			outputPaths[i] = fmt.Sprintf("lumberjack://%s", v)
+		}
+	}
+
+	if logRotationEnable != 0 {
+		setupLogRotation(logRotateConfigJSON)
+	}
+
+	config := loadConfig(level)
+	config.OutputPaths = outputPaths
+	p, err := config.Build(zap.AddCallerSkip(1))
+	FatalIfError(err)
+	logger = p.Sugar()
+}
+
+type lumberjackSink struct {
+	lumberjack.Logger
+}
+
+func (*lumberjackSink) Sync() error {
+	return nil
+}
+
+// setupLogRotation initializes log rotation for a single file path target.
+func setupLogRotation(logRotateConfigJSON string) {
+	err := zap.RegisterSink("lumberjack", func(u *url.URL) (zap.Sink, error) {
+		var conf lumberjackSink
+		err := json.Unmarshal([]byte(logRotateConfigJSON), &conf)
+		FatalfIf(err != nil, "bad config LogRotateConfigJSON: %v", err)
+		conf.Filename = u.Host + u.Path
+		return &conf, nil
+	})
+	FatalIfError(err)
+}
+
+func loadConfig(logLevel string) zap.Config {
 	config := zap.NewProductionConfig()
-	err := config.Level.UnmarshalText([]byte(level))
+	err := config.Level.UnmarshalText([]byte(logLevel))
 	FatalIfError(err)
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	if os.Getenv("DTM_DEBUG") != "" {
 		config.Encoding = "console"
 		config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
 	}
-	p, err := config.Build(zap.AddCallerSkip(1))
-	FatalIfError(err)
-	logger = p.Sugar()
+	return config
 }
 
 // Debugf log to level debug

dtmcli/logger/logger_test.go

@@ -16,6 +16,15 @@ func TestInitLog(t *testing.T) {
 	Errorf("a error msg")
 	FatalfIf(false, "nothing")
 	FatalIfError(nil)
+
+	InitLog2("debug", "test.log|stderr", 0, "")
+	Debugf("a debug msg to console and file")
+
+	InitLog2("debug", "test2.log|/tmp/dtm-test1.log|/tmp/dtm-test.log|stdout|stderr", 1,
+		"{\"maxsize\": 1, \"maxage\": 1, \"maxbackups\": 1, \"compress\": false}")
+	Debugf("a debug msg to /tmp/dtm-test.log|test2.log|stdout|stderr")
+
+	// _ = os.Remove("test.log")
 }
 
 func TestWithLogger(t *testing.T) {

dtmsvr/config/config.go

@@ -29,6 +29,13 @@ type MicroService struct {
 	EndPoint string `yaml:"EndPoint"`
 }
 
+// Log config customize log
+type Log struct {
+	Outputs            string `yaml:"Outputs" default:"stderr"`
+	RotationEnable     int64  `yaml:"RotationEnable" default:"0"`
+	RotationConfigJSON string `yaml:"RotationConfigJSON" default:"{}"`
+}
+
 // Store defines storage relevant info
 type Store struct {
 	Driver             string `yaml:"Driver" default:"boltdb"`
@@ -73,6 +80,7 @@ type configType struct {
 	UpdateBranchSync              int64        `yaml:"UpdateBranchSync"`
 	UpdateBranchAsyncGoroutineNum int64        `yaml:"UpdateBranchAsyncGoroutineNum" default:"1"`
 	LogLevel                      string       `yaml:"LogLevel" default:"info"`
+	Log                           Log          `yaml:"Log"`
 }
 
 // Config 配置

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/lib/pq v1.10.3
 	github.com/lithammer/shortuuid v2.0.3+incompatible
 	github.com/lithammer/shortuuid/v3 v3.0.7
+	github.com/natefinch/lumberjack v2.0.0+incompatible // indirect
 	github.com/onsi/gomega v1.16.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/stretchr/testify v1.7.0

main.go

@@ -18,7 +18,6 @@ import (
 	"github.com/dtm-labs/dtm/dtmsvr"
 	"github.com/dtm-labs/dtm/dtmsvr/config"
 	"github.com/dtm-labs/dtm/dtmsvr/storage/registry"
-	"github.com/natefinch/lumberjack"
 
 	// load the microserver driver
 	_ "github.com/dtm-labs/dtmdriver-gozero"
@@ -61,21 +60,9 @@ func main() {
 	config.MustLoadConfig(*confFile)
 	conf := &config.Config
 	if *isDebug {
-		config.Config.LogLevel = "debug"
+		conf.LogLevel = "debug"
 	}
-	if conf.Log.Output == "file" {
-		ll := lumberjack.Logger{
-			Filename:   conf.Log.FileName,
-			MaxSize:    int(conf.Log.FileMaxSize),
-			MaxBackups: int(conf.Log.FileMaxBackups),
-			MaxAge:     int(conf.Log.FileMaxAge),
-			Compress:   conf.Log.FileCompress != 0,
-		}
-		logger.InitRotateLog(conf.Log.Level, &ll)
-	} else {
-		logger.InitLog(conf.Log.Level)
-	}
-
+	logger.InitLog2(conf.LogLevel, conf.Log.Outputs, conf.Log.RotationEnable, conf.Log.RotationConfigJSON)
 	if *isReset {
 		dtmsvr.PopulateDB(false)
 	}