From 8e743314e3dc4676b71a727c0e88bb5621c8eb97 Mon Sep 17 00:00:00 2001 From: tian Date: Mon, 17 Jan 2022 00:11:29 +0800 Subject: [PATCH 1/4] add log config for output to file. --- bench/main.go | 2 +- conf.sample.yml | 10 +++++++++- dtmcli/logger/log.go | 33 +++++++++++++++++++++++++++++++++ dtmsvr/config/config.go | 13 ++++++++++++- dtmsvr/svr.go | 11 +++++++++++ go.mod | 1 + main.go | 2 +- 7 files changed, 68 insertions(+), 4 deletions(-) diff --git a/bench/main.go b/bench/main.go index 34bd859..50e488b 100644 --- a/bench/main.go +++ b/bench/main.go @@ -33,7 +33,7 @@ func main() { } logger.Infof("starting bench server") config.MustLoadConfig("") - logger.InitLog(conf.LogLevel) + logger.InitLog(conf.Log.Level) if busi.BusiConf.Driver != "" { dtmcli.SetCurrentDBType(busi.BusiConf.Driver) svr.PrepareBenchDB() diff --git a/conf.sample.yml b/conf.sample.yml index cd66cb6..5c046ed 100644 --- a/conf.sample.yml +++ b/conf.sample.yml @@ -48,7 +48,15 @@ # RetryInterval: 10 # the subtrans branch will be retried after this interval # RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm -# LogLevel: 'info' # default: info. can be debug|info|warn|error +# Log: +# Level: 'info' # default: info. can be debug|info|warn|error +# Output: 'console' # default: console. can be console|file +# FileName: '/tmp/dtm.log' # default: /tmp/dtm.log. +# FileMaxSize: 10 # default: 10, unit: MB. +# FileMaxBackups: 5 # default: 5. +# FileMaxAge: 30 # default: 30, unit: days. +# FileCompress: 0 # default: 0. can by 0|1, means false|true + # HttpPort: 36789 # GrpcPort: 36790 diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 59b3d43..54237df 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -1,9 +1,12 @@ package logger import ( + "fmt" "log" + "net/url" "os" + "github.com/natefinch/lumberjack" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -24,6 +27,14 @@ type Logger interface { Errorf(format string, args ...interface{}) } +type lumberjackSink struct { + *lumberjack.Logger +} + +func (lumberjackSink) Sync() error { + return nil +} + // WithLogger replaces default logger func WithLogger(log Logger) { logger = log @@ -45,6 +56,28 @@ func InitLog(level string) { logger = p.Sugar() } +// InitRotateLog is an initialization for a rotated logger by lumberjack +func InitRotateLog(logLevel string, output, logFile string, ll *lumberjack.Logger) { + config := zap.NewProductionConfig() + err := config.Level.UnmarshalText([]byte(logLevel)) + FatalIfError(err) + config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + config.Encoding = "console" + config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + config.OutputPaths = []string{fmt.Sprintf("lumberjack:%s", logFile), "stdout"} + if output == "file" { + err := zap.RegisterSink("lumberjack", func(*url.URL) (zap.Sink, error) { + return lumberjackSink{ + Logger: ll, + }, nil + }) + FatalIfError(err) + } + p, err := config.Build(zap.AddCallerSkip(1)) + FatalIfError(err) + logger = p.Sugar() +} + // Debugf log to level debug func Debugf(fmt string, args ...interface{}) { logger.Debugf(fmt, args...) diff --git a/dtmsvr/config/config.go b/dtmsvr/config/config.go index 2e84887..cc74975 100644 --- a/dtmsvr/config/config.go +++ b/dtmsvr/config/config.go @@ -29,6 +29,17 @@ type MicroService struct { EndPoint string `yaml:"EndPoint"` } +// Log config customize log +type Log struct { + Level string `yaml:"Level" default:"info"` + Output string `yaml:"Output" default:"console"` + FileName string `yaml:"FileName" default:"/tmp/dtm.log"` + FileMaxSize int64 `yaml:"FileMaxSize" default:"10"` + FileMaxBackups int64 `yaml:"FileMaxBackups" default:"5"` + FileMaxAge int64 `yaml:"FileMaxAge" default:"30"` + FileCompress int64 `yaml:"FileCompress" default:"0"` +} + // Store defines storage relevant info type Store struct { Driver string `yaml:"Driver" default:"boltdb"` @@ -72,7 +83,7 @@ type configType struct { MicroService MicroService `yaml:"MicroService"` UpdateBranchSync int64 `yaml:"UpdateBranchSync"` UpdateBranchAsyncGoroutineNum int64 `yaml:"UpdateBranchAsyncGoroutineNum" default:"1"` - LogLevel string `yaml:"LogLevel" default:"info"` + Log Log `yaml:"Log"` } // Config 配įŊŽ diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index 56f288c..0470f0a 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -20,12 +20,23 @@ import ( "github.com/dtm-labs/dtm/dtmutil" "github.com/dtm-labs/dtmdriver" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + "github.com/natefinch/lumberjack" "google.golang.org/grpc" ) // StartSvr StartSvr func StartSvr() { logger.Infof("start dtmsvr") + if conf.Log.Output == "file" { + ll := lumberjack.Logger{ + Filename: conf.Log.FileName, + MaxSize: int(conf.Log.FileMaxSize), + MaxBackups: int(conf.Log.FileMaxBackups), + MaxAge: int(conf.Log.FileMaxAge), + Compress: conf.Log.FileCompress != 0, + } + logger.InitRotateLog(conf.Log.Level, conf.Log.Output, conf.Log.FileName, &ll) + } dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second) diff --git a/go.mod b/go.mod index ef2be14..baf57ea 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/lib/pq v1.10.3 github.com/lithammer/shortuuid v2.0.3+incompatible github.com/lithammer/shortuuid/v3 v3.0.7 + github.com/natefinch/lumberjack v2.0.0+incompatible // indirect github.com/onsi/gomega v1.16.0 github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 diff --git a/main.go b/main.go index 1b088e1..43ce895 100644 --- a/main.go +++ b/main.go @@ -59,7 +59,7 @@ func main() { } config.MustLoadConfig(*confFile) if *isDebug { - config.Config.LogLevel = "debug" + config.Config.Log.Level = "debug" } if *isReset { dtmsvr.PopulateDB(false) From 5d280c31db822a0a189cc711c571ff1453521944 Mon Sep 17 00:00:00 2001 From: tian Date: Mon, 17 Jan 2022 00:41:22 +0800 Subject: [PATCH 2/4] add unit test for log. --- dtmcli/logger/log.go | 43 ++++++++++++++++++------------------ dtmcli/logger/logger_test.go | 20 +++++++++++++++++ dtmsvr/svr.go | 2 +- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/dtmcli/logger/log.go b/dtmcli/logger/log.go index 54237df..e2ef110 100644 --- a/dtmcli/logger/log.go +++ b/dtmcli/logger/log.go @@ -43,39 +43,38 @@ func WithLogger(log Logger) { // InitLog is an initialization for a logger // level can be: debug info warn error func InitLog(level string) { - config := zap.NewProductionConfig() - err := config.Level.UnmarshalText([]byte(level)) - FatalIfError(err) - config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - if os.Getenv("DTM_DEBUG") != "" { - config.Encoding = "console" - config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - } + config := loadConfig(level) p, err := config.Build(zap.AddCallerSkip(1)) FatalIfError(err) logger = p.Sugar() } // InitRotateLog is an initialization for a rotated logger by lumberjack -func InitRotateLog(logLevel string, output, logFile string, ll *lumberjack.Logger) { +func InitRotateLog(logLevel string, ll *lumberjack.Logger) { + config := loadConfig(logLevel) + config.OutputPaths = []string{fmt.Sprintf("lumberjack:%s", ll.Filename), "stdout"} + err := zap.RegisterSink("lumberjack", func(*url.URL) (zap.Sink, error) { + return lumberjackSink{ + Logger: ll, + }, nil + }) + FatalIfError(err) + + p, err := config.Build(zap.AddCallerSkip(1)) + FatalIfError(err) + logger = p.Sugar() +} + +func loadConfig(logLevel string) zap.Config { config := zap.NewProductionConfig() err := config.Level.UnmarshalText([]byte(logLevel)) FatalIfError(err) config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - config.Encoding = "console" - config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - config.OutputPaths = []string{fmt.Sprintf("lumberjack:%s", logFile), "stdout"} - if output == "file" { - err := zap.RegisterSink("lumberjack", func(*url.URL) (zap.Sink, error) { - return lumberjackSink{ - Logger: ll, - }, nil - }) - FatalIfError(err) + if os.Getenv("DTM_DEBUG") != "" { + config.Encoding = "console" + config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder } - p, err := config.Build(zap.AddCallerSkip(1)) - FatalIfError(err) - logger = p.Sugar() + return config } // Debugf log to level debug diff --git a/dtmcli/logger/logger_test.go b/dtmcli/logger/logger_test.go index 26797e0..bcba763 100644 --- a/dtmcli/logger/logger_test.go +++ b/dtmcli/logger/logger_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "github.com/natefinch/lumberjack" "go.uber.org/zap" ) @@ -28,3 +29,22 @@ func TestWithLogger(t *testing.T) { FatalfIf(false, "nothing") FatalIfError(nil) } + +func TestInitRotateLog(t *testing.T) { + os.Setenv("DTM_DEBUG", "1") + ll := lumberjack.Logger{ + Filename: "test.log", + MaxSize: 1, + MaxBackups: 1, + MaxAge: 1, + Compress: false, + } + InitRotateLog("debug", &ll) + Debugf("a debug msg") + Infof("a info msg") + Warnf("a warn msg") + Errorf("a error msg") + FatalfIf(false, "nothing") + FatalIfError(nil) + _ = os.Remove("test.log") +} diff --git a/dtmsvr/svr.go b/dtmsvr/svr.go index 0470f0a..8f25501 100644 --- a/dtmsvr/svr.go +++ b/dtmsvr/svr.go @@ -35,7 +35,7 @@ func StartSvr() { MaxAge: int(conf.Log.FileMaxAge), Compress: conf.Log.FileCompress != 0, } - logger.InitRotateLog(conf.Log.Level, conf.Log.Output, conf.Log.FileName, &ll) + logger.InitRotateLog(conf.Log.Level, &ll) } dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second) dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { From 4c903aed6acf95d089895019d8017eb5dabc3a30 Mon Sep 17 00:00:00 2001 From: tina <11660793+tina-prog@users.noreply.github.com> Date: Mon, 17 Jan 2022 01:31:32 +0800 Subject: [PATCH 3/4] Create codeql-analysis1.yml --- .github/workflows/codeql-analysis1.yml | 70 ++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/codeql-analysis1.yml diff --git a/.github/workflows/codeql-analysis1.yml b/.github/workflows/codeql-analysis1.yml new file mode 100644 index 0000000..5eb581e --- /dev/null +++ b/.github/workflows/codeql-analysis1.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '19 3 * * 2' + workflow_dispatch: + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 2971fbf2840961e852ef699aa82c27046e7c2d0a Mon Sep 17 00:00:00 2001 From: tian Date: Mon, 17 Jan 2022 01:57:33 +0800 Subject: [PATCH 4/4] delete useless file --- .github/workflows/codeql-analysis1.yml | 70 -------------------------- 1 file changed, 70 deletions(-) delete mode 100644 .github/workflows/codeql-analysis1.yml diff --git a/.github/workflows/codeql-analysis1.yml b/.github/workflows/codeql-analysis1.yml deleted file mode 100644 index 5eb581e..0000000 --- a/.github/workflows/codeql-analysis1.yml +++ /dev/null @@ -1,70 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - schedule: - - cron: '19 3 * * 2' - workflow_dispatch: - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1