diff --git a/dtmsvr/trans_type_workflow.go b/dtmsvr/trans_type_workflow.go index 925323c..3e96c35 100644 --- a/dtmsvr/trans_type_workflow.go +++ b/dtmsvr/trans_type_workflow.go @@ -2,11 +2,15 @@ package dtmsvr import ( "context" + "errors" + "math" "github.com/dtm-labs/dtm/client/dtmcli" "github.com/dtm-labs/dtm/client/dtmcli/dtmimp" "github.com/dtm-labs/dtm/client/dtmgrpc/dtmgimp" "github.com/dtm-labs/dtm/client/workflow/wfpb" + "github.com/dtm-labs/logger" + "github.com/gin-gonic/gin" ) type transWorkflowProcessor struct { @@ -38,5 +42,26 @@ func (t *transWorkflowProcessor) ProcessOnce(ctx context.Context, branches []Tra wd := wfpb.WorkflowData{Data: cmc.Data} data = dtmgimp.MustProtoMarshal(&wd) } - return t.getURLResult(ctx, t.QueryPrepared, "00", cmc.Name, data) + err := t.getURLResult(ctx, t.QueryPrepared, "00", cmc.Name, data) + if err == dtmimp.ErrOngoing { + t.touchCronTime(cronKeep, 0) + } else if err != nil && !errors.Is(err, dtmimp.ErrFailure) { + t.touchCronTime(cronBackoff, 0) + v := t.NextCronInterval / t.getNextCronInterval(cronReset) + retryCount := int64(math.Log2(float64(v))) + logger.Debugf("origin: %d v: %d retryCount: %d", t.getNextCronInterval(cronReset), v, retryCount) + if retryCount >= conf.AlertRetryLimit && conf.AlertWebHook != "" { + _, err2 := dtmcli.GetRestyClient().R().SetBody(gin.H{ + "gid": t.Gid, + "status": t.Status, + "branch": "", + "error": err.Error(), + "retry_count": retryCount, + }).Post(conf.AlertWebHook) + if err2 != nil { + logger.Errorf("alerting webhook error: %v", err2) + } + } + } + return err }