Merge pull request #328 from dtm-labs/alpha

Support workflow pattern

dtmcli/dtmimp/trans_base.go

@@ -50,7 +50,6 @@ type TransOptions struct {
 	PassthroughHeaders []string          `json:"passthrough_headers,omitempty" gorm:"-"` // for inherit the specified gin context headers
 	BranchHeaders      map[string]string `json:"branch_headers,omitempty" gorm:"-"`      // custom branch headers,  dtm server => service api
 	Concurrent         bool              `json:"concurrent" gorm:"-"`                    // for trans type: saga msg
-	RollbackReason     string            `json:"rollback_reason,omitempty" gorm:"-"`
 }
 
 // TransBase base for all trans
@@ -68,8 +67,9 @@ type TransBase struct {
 	BranchIDGen `json:"-"`          // used in XA/TCC
 	Op          string              `json:"-"` // used in XA/TCC
 
-	QueryPrepared string `json:"query_prepared,omitempty"` // used in MSG
+	QueryPrepared  string `json:"query_prepared,omitempty"` // used in MSG
-	Protocol      string `json:"protocol"`
+	Protocol       string `json:"protocol"`
+	RollbackReason string `json:"rollback_reason,omitempty" gorm:"-"`
 }
 
 // NewTransBase new a TransBase
@@ -94,39 +94,29 @@ func TransBaseFromQuery(qs url.Values) *TransBase {
 	return NewTransBase(EscapeGet(qs, "gid"), EscapeGet(qs, "trans_type"), EscapeGet(qs, "dtm"), EscapeGet(qs, "branch_id"))
 }
 
-// TransCallDtm TransBase call dtm
+// TransCallDtmExt TransBase call dtm
-func TransCallDtm(tb *TransBase, body interface{}, operation string) error {
+func TransCallDtmExt(tb *TransBase, body interface{}, operation string) (*resty.Response, error) {
+	if tb.Protocol == Jrpc {
+		return transCallDtmJrpc(tb, body, operation)
+	}
 	if tb.RequestTimeout != 0 {
 		RestyClient.SetTimeout(time.Duration(tb.RequestTimeout) * time.Second)
 	}
-	if tb.Protocol == Jrpc {
-		var result map[string]interface{}
-		resp, err := RestyClient.R().
-			SetBody(map[string]interface{}{
-				"jsonrpc": "2.0",
-				"id":      "no-use",
-				"method":  operation,
-				"params":  body,
-			}).
-			SetResult(&result).
-			Post(tb.Dtm)
-		if err != nil {
-			return err
-		}
-		if resp.StatusCode() != http.StatusOK || result["error"] != nil {
-			return errors.New(resp.String())
-		}
-		return nil
-	}
 	resp, err := RestyClient.R().
 		SetBody(body).Post(fmt.Sprintf("%s/%s", tb.Dtm, operation))
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if resp.StatusCode() != http.StatusOK || strings.Contains(resp.String(), ResultFailure) {
-		return errors.New(resp.String())
+		return nil, errors.New(resp.String())
 	}
-	return nil
+	return resp, nil
+}
+
+// TransCallDtm is the short call for TransCallDtmExt
+func TransCallDtm(tb *TransBase, operation string) error {
+	_, err := TransCallDtmExt(tb, tb, operation)
+	return err
 }
 
 // TransRegisterBranch TransBase register a branch to dtm
@@ -138,7 +128,8 @@ func TransRegisterBranch(tb *TransBase, added map[string]string, operation strin
 	for k, v := range added {
 		m[k] = v
 	}
-	return TransCallDtm(tb, m, operation)
+	_, err := TransCallDtmExt(tb, m, operation)
+	return err
 }
 
 // TransRequestBranch TransBase request branch result
@@ -166,3 +157,26 @@ func TransRequestBranch(t *TransBase, method string, body interface{}, branchID
 	}
 	return resp, err
 }
+
+func transCallDtmJrpc(tb *TransBase, body interface{}, operation string) (*resty.Response, error) {
+	if tb.RequestTimeout != 0 {
+		RestyClient.SetTimeout(time.Duration(tb.RequestTimeout) * time.Second)
+	}
+	var result map[string]interface{}
+	resp, err := RestyClient.R().
+		SetBody(map[string]interface{}{
+			"jsonrpc": "2.0",
+			"id":      "no-use",
+			"method":  operation,
+			"params":  body,
+		}).
+		SetResult(&result).
+		Post(tb.Dtm)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode() != http.StatusOK || result["error"] != nil {
+		return nil, errors.New(resp.String())
+	}
+	return resp, nil
+}

dtmcli/dtmimp/vars.go

@@ -42,17 +42,23 @@ var PassthroughHeaders = []string{}
 // BarrierTableName the table name of barrier table
 var BarrierTableName = "dtm_barrier.barrier"
 
+// BeforeRequest is the middleware for default resty.Client
+func BeforeRequest(c *resty.Client, r *resty.Request) error {
+	r.URL = MayReplaceLocalhost(r.URL)
+	u, err := dtmdriver.GetHTTPDriver().ResolveURL(r.URL)
+	logger.Debugf("requesting: %s %s %s resolved: %s", r.Method, r.URL, MustMarshalString(r.Body), u)
+	r.URL = u
+	return err
+}
+
+// AfterResponse is the middleware for default resty.Client
+func AfterResponse(c *resty.Client, resp *resty.Response) error {
+	r := resp.Request
+	logger.Debugf("requested: %d %s %s %s", resp.StatusCode(), r.Method, r.URL, resp.String())
+	return nil
+}
+
 func init() {
-	RestyClient.OnBeforeRequest(func(c *resty.Client, r *resty.Request) error {
+	RestyClient.OnBeforeRequest(BeforeRequest)
-		r.URL = MayReplaceLocalhost(r.URL)
+	RestyClient.OnAfterResponse(AfterResponse)
-		u, err := dtmdriver.GetHTTPDriver().ResolveURL(r.URL)
-		logger.Debugf("requesting: %s %s %s resolved: %s", r.Method, r.URL, MustMarshalString(r.Body), u)
-		r.URL = u
-		return err
-	})
-	RestyClient.OnAfterResponse(func(c *resty.Client, resp *resty.Response) error {
-		r := resp.Request
-		logger.Debugf("requested: %s %s %s", r.Method, r.URL, resp.String())
-		return nil
-	})
 }

dtmcli/msg.go → dtmcli/trans_msg.go

@@ -40,13 +40,13 @@ func (s *Msg) SetDelay(delay uint64) *Msg {
 // Prepare prepare the msg, msg will later be submitted
 func (s *Msg) Prepare(queryPrepared string) error {
 	s.QueryPrepared = dtmimp.OrString(queryPrepared, s.QueryPrepared)
-	return dtmimp.TransCallDtm(&s.TransBase, s, "prepare")
+	return dtmimp.TransCallDtm(&s.TransBase, "prepare")
 }
 
 // Submit submit the msg
 func (s *Msg) Submit() error {
 	s.BuildCustomOptions()
-	return dtmimp.TransCallDtm(&s.TransBase, s, "submit")
+	return dtmimp.TransCallDtm(&s.TransBase, "submit")
 }
 
 // DoAndSubmitDB short method for Do on db type. please see DoAndSubmit
@@ -72,7 +72,7 @@ func (s *Msg) DoAndSubmit(queryPrepared string, busiCall func(bb *BranchBarrier)
 			_, err = dtmimp.TransRequestBranch(&s.TransBase, "GET", nil, bb.BranchID, bb.Op, queryPrepared)
 		}
 		if errors.Is(errb, ErrFailure) || errors.Is(err, ErrFailure) {
-			_ = dtmimp.TransCallDtm(&s.TransBase, s, "abort")
+			_ = dtmimp.TransCallDtm(&s.TransBase, "abort")
 		} else if err == nil {
 			err = s.Submit()
 		}

dtmcli/saga.go → dtmcli/trans_saga.go

@@ -43,7 +43,7 @@ func (s *Saga) SetConcurrent() *Saga {
 // Submit submit the saga trans
 func (s *Saga) Submit() error {
 	s.BuildCustomOptions()
-	return dtmimp.TransCallDtm(&s.TransBase, s, "submit")
+	return dtmimp.TransCallDtm(&s.TransBase, "submit")
 }
 
 // BuildCustomOptions add custom options to the request context

dtmcli/tcc.go → dtmcli/trans_tcc.go

@@ -34,17 +34,17 @@ func TccGlobalTransaction(dtm string, gid string, tccFunc TccGlobalFunc) (rerr e
 func TccGlobalTransaction2(dtm string, gid string, custom func(*Tcc), tccFunc TccGlobalFunc) (rerr error) {
 	tcc := &Tcc{TransBase: *dtmimp.NewTransBase(gid, "tcc", dtm, "")}
 	custom(tcc)
-	rerr = dtmimp.TransCallDtm(&tcc.TransBase, tcc, "prepare")
+	rerr = dtmimp.TransCallDtm(&tcc.TransBase, "prepare")
 	if rerr != nil {
 		return rerr
 	}
 	defer dtmimp.DeferDo(&rerr, func() error {
-		return dtmimp.TransCallDtm(&tcc.TransBase, tcc, "submit")
+		return dtmimp.TransCallDtm(&tcc.TransBase, "submit")
 	}, func() error {
 		if rerr != nil {
 			tcc.RollbackReason = rerr.Error()
 		}
-		return dtmimp.TransCallDtm(&tcc.TransBase, tcc, "abort")
+		return dtmimp.TransCallDtm(&tcc.TransBase, "abort")
 	})
 	_, rerr = tccFunc(tcc)
 	return

dtmcli/types.go

@@ -7,24 +7,10 @@
 package dtmcli
 
 import (
-	"errors"
-	"fmt"
-	"net/http"
-
 	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
 	"github.com/go-resty/resty/v2"
 )
 
-// MustGenGid generate a new gid
-func MustGenGid(server string) string {
-	res := map[string]string{}
-	resp, err := dtmimp.RestyClient.R().SetResult(&res).Get(server + "/newGid")
-	if err != nil || res["gid"] == "" {
-		panic(fmt.Errorf("newGid error: %v, resp: %s", err, resp))
-	}
-	return res["gid"]
-}
-
 // DB interface
 type DB = dtmimp.DB
 
@@ -34,16 +20,6 @@ type TransOptions = dtmimp.TransOptions
 // DBConf declares db configuration
 type DBConf = dtmimp.DBConf
 
-// String2DtmError translate string to dtm error
-func String2DtmError(str string) error {
-	return map[string]error{
-		ResultFailure: ErrFailure,
-		ResultOngoing: ErrOngoing,
-		ResultSuccess: nil,
-		"":            nil,
-	}[str]
-}
-
 // SetCurrentDBType set currentDBType
 func SetCurrentDBType(dbType string) {
 	dtmimp.SetCurrentDBType(dbType)
@@ -81,25 +57,3 @@ func GetRestyClient() *resty.Client {
 func SetPassthroughHeaders(headers []string) {
 	dtmimp.PassthroughHeaders = headers
 }
-
-// Result2HttpJSON return the http code and json result
-// if result is error, the return proper code, else return StatusOK
-func Result2HttpJSON(result interface{}) (code int, res interface{}) {
-	err, _ := result.(error)
-	if err == nil {
-		code = http.StatusOK
-		res = result
-	} else {
-		res = map[string]string{
-			"error": err.Error(),
-		}
-		if errors.Is(err, ErrFailure) {
-			code = http.StatusConflict
-		} else if errors.Is(err, ErrOngoing) {
-			code = http.StatusTooEarly
-		} else if err != nil {
-			code = http.StatusInternalServerError
-		}
-	}
-	return
-}

dtmcli/utils.go

@@ -0,0 +1,62 @@
+package dtmcli
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/go-resty/resty/v2"
+)
+
+// MustGenGid generate a new gid
+func MustGenGid(server string) string {
+	res := map[string]string{}
+	resp, err := dtmimp.RestyClient.R().SetResult(&res).Get(server + "/newGid")
+	if err != nil || res["gid"] == "" {
+		panic(fmt.Errorf("newGid error: %v, resp: %s", err, resp))
+	}
+	return res["gid"]
+}
+
+// String2DtmError translate string to dtm error
+func String2DtmError(str string) error {
+	return map[string]error{
+		ResultFailure: ErrFailure,
+		ResultOngoing: ErrOngoing,
+		ResultSuccess: nil,
+		"":            nil,
+	}[str]
+}
+
+// Result2HttpJSON return the http code and json result
+// if result is error, the return proper code, else return StatusOK
+func Result2HttpJSON(result interface{}) (code int, res interface{}) {
+	err, _ := result.(error)
+	if err == nil {
+		code = http.StatusOK
+		res = result
+	} else {
+		res = map[string]string{
+			"error": err.Error(),
+		}
+		if errors.Is(err, ErrFailure) {
+			code = http.StatusConflict
+		} else if errors.Is(err, ErrOngoing) {
+			code = http.StatusTooEarly
+		} else if err != nil {
+			code = http.StatusInternalServerError
+		}
+	}
+	return
+}
+
+// IsRollback returns whether the result is indicating rollback
+func IsRollback(resp *resty.Response, err error) bool {
+	return err == ErrFailure || dtmimp.RespAsErrorCompatible(resp) == ErrFailure
+}
+
+// IsOngoing returns whether the result is indicating ongoing
+func IsOngoing(resp *resty.Response, err error) bool {
+	return err == ErrOngoing || dtmimp.RespAsErrorCompatible(resp) == ErrOngoing
+}

dtmcli/xa.go

@@ -69,7 +69,7 @@ func XaGlobalTransaction2(server string, gid string, custom func(*Xa), xaFunc Xa
 	xa := &Xa{TransBase: *dtmimp.NewTransBase(gid, "xa", server, "")}
 	custom(xa)
 	return dtmimp.XaHandleGlobalTrans(&xa.TransBase, func(action string) error {
-		return dtmimp.TransCallDtm(&xa.TransBase, xa, action)
+		return dtmimp.TransCallDtm(&xa.TransBase, action)
 	}, func() error {
 		_, rerr := xaFunc(xa)
 		return rerr

dtmgrpc/dtmgimp/types.go

@@ -15,7 +15,9 @@ import (
 	"github.com/dtm-labs/dtm/dtmcli/logger"
 	"github.com/dtm-labs/dtmdriver"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -27,10 +29,11 @@ func GrpcServerLog(ctx context.Context, req interface{}, info *grpc.UnaryServerI
 	m, err := handler(ctx, req)
 	res := fmt.Sprintf("%2dms %v %s %s %s",
 		time.Since(began).Milliseconds(), err, info.FullMethod, dtmimp.MustMarshalString(m), dtmimp.MustMarshalString(req))
-	if err != nil {
+	st, _ := status.FromError(err)
-		logger.Errorf("%s", res)
+	if err == nil || st != nil && st.Code() == codes.FailedPrecondition {
-	} else {
 		logger.Infof("%s", res)
+	} else {
+		logger.Errorf("%s", res)
 	}
 	return m, err
 }
@@ -42,10 +45,11 @@ func GrpcClientLog(ctx context.Context, method string, req, reply interface{}, c
 	err := invoker(ctx, method, req, reply, cc, opts...)
 	res := fmt.Sprintf("grpc client called: %s%s %s result: %s err: %v",
 		cc.Target(), method, dtmimp.MustMarshalString(req), dtmimp.MustMarshalString(reply), err)
-	if err != nil {
+	st, _ := status.FromError(err)
-		logger.Errorf("%s", res)
+	if err == nil || st != nil && st.Code() == codes.FailedPrecondition {
+		logger.Infof("%s", res)
 	} else {
-		logger.Debugf("%s", res)
+		logger.Errorf("%s", res)
 	}
 	return err
 }

dtmgrpc/dtmgimp/utils.go

@@ -24,10 +24,15 @@ func MustProtoMarshal(msg proto.Message) []byte {
 	return b
 }
 
-// DtmGrpcCall make a convenient call to dtm
+// MustProtoUnmarshal must version of proto.Unmarshal
-func DtmGrpcCall(s *dtmimp.TransBase, operation string) error {
+func MustProtoUnmarshal(data []byte, msg proto.Message) {
-	reply := emptypb.Empty{}
+	err := proto.Unmarshal(data, msg)
-	return MustGetGrpcConn(s.Dtm, false).Invoke(s.Context, "/dtmgimp.Dtm/"+operation, &dtmgpb.DtmRequest{
+	dtmimp.PanicIf(err != nil, err)
+}
+
+// GetDtmRequest return a DtmRequest from TransBase
+func GetDtmRequest(s *dtmimp.TransBase) *dtmgpb.DtmRequest {
+	return &dtmgpb.DtmRequest{
 		Gid:       s.Gid,
 		TransType: s.TransType,
 		TransOptions: &dtmgpb.DtmTransOptions{
@@ -37,13 +42,19 @@ func DtmGrpcCall(s *dtmimp.TransBase, operation string) error {
 			PassthroughHeaders: s.PassthroughHeaders,
 			BranchHeaders:      s.BranchHeaders,
 			RequestTimeout:     s.RequestTimeout,
-			RollbackReason:     s.RollbackReason,
 		},
-		QueryPrepared: s.QueryPrepared,
+		QueryPrepared:  s.QueryPrepared,
-		CustomedData:  s.CustomData,
+		CustomedData:   s.CustomData,
-		BinPayloads:   s.BinPayloads,
+		BinPayloads:    s.BinPayloads,
-		Steps:         dtmimp.MustMarshalString(s.Steps),
+		Steps:          dtmimp.MustMarshalString(s.Steps),
-	}, &reply)
+		RollbackReason: s.RollbackReason,
+	}
+}
+
+// DtmGrpcCall make a convenient call to dtm
+func DtmGrpcCall(s *dtmimp.TransBase, operation string) error {
+	reply := emptypb.Empty{}
+	return MustGetGrpcConn(s.Dtm, false).Invoke(s.Context, "/dtmgimp.Dtm/"+operation, GetDtmRequest(s), &reply)
 }
 
 const dtmpre string = "dtm-"

dtmgrpc/dtmgpb/dtmgimp.pb.go

@@ -32,7 +32,6 @@ type DtmTransOptions struct {
 	PassthroughHeaders []string          `protobuf:"bytes,4,rep,name=PassthroughHeaders,proto3" json:"PassthroughHeaders,omitempty"`
 	BranchHeaders      map[string]string `protobuf:"bytes,5,rep,name=BranchHeaders,proto3" json:"BranchHeaders,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	RequestTimeout     int64             `protobuf:"varint,6,opt,name=RequestTimeout,proto3" json:"RequestTimeout,omitempty"`
-	RollbackReason     string            `protobuf:"bytes,7,opt,name=RollbackReason,proto3" json:"RollbackReason,omitempty"`
 }
 
 func (x *DtmTransOptions) Reset() {
@@ -109,26 +108,21 @@ func (x *DtmTransOptions) GetRequestTimeout() int64 {
 	return 0
 }
 
-func (x *DtmTransOptions) GetRollbackReason() string {
-	if x != nil {
-		return x.RollbackReason
-	}
-	return ""
-}
-
 // DtmRequest request sent to dtm server
 type DtmRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Gid           string           `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
+	Gid            string            `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
-	TransType     string           `protobuf:"bytes,2,opt,name=TransType,proto3" json:"TransType,omitempty"`
+	TransType      string            `protobuf:"bytes,2,opt,name=TransType,proto3" json:"TransType,omitempty"`
-	TransOptions  *DtmTransOptions `protobuf:"bytes,3,opt,name=TransOptions,proto3" json:"TransOptions,omitempty"`
+	TransOptions   *DtmTransOptions  `protobuf:"bytes,3,opt,name=TransOptions,proto3" json:"TransOptions,omitempty"`
-	CustomedData  string           `protobuf:"bytes,4,opt,name=CustomedData,proto3" json:"CustomedData,omitempty"`
+	CustomedData   string            `protobuf:"bytes,4,opt,name=CustomedData,proto3" json:"CustomedData,omitempty"`
-	BinPayloads   [][]byte         `protobuf:"bytes,5,rep,name=BinPayloads,proto3" json:"BinPayloads,omitempty"`     // for MSG/SAGA branch payloads
+	BinPayloads    [][]byte          `protobuf:"bytes,5,rep,name=BinPayloads,proto3" json:"BinPayloads,omitempty"`     // for Msg/Saga/Workflow branch payloads
-	QueryPrepared string           `protobuf:"bytes,6,opt,name=QueryPrepared,proto3" json:"QueryPrepared,omitempty"` // for MSG
+	QueryPrepared  string            `protobuf:"bytes,6,opt,name=QueryPrepared,proto3" json:"QueryPrepared,omitempty"` // for Msg
-	Steps         string           `protobuf:"bytes,7,opt,name=Steps,proto3" json:"Steps,omitempty"`
+	Steps          string            `protobuf:"bytes,7,opt,name=Steps,proto3" json:"Steps,omitempty"`
+	ReqExtra       map[string]string `protobuf:"bytes,8,rep,name=ReqExtra,proto3" json:"ReqExtra,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	RollbackReason string            `protobuf:"bytes,9,opt,name=RollbackReason,proto3" json:"RollbackReason,omitempty"`
 }
 
 func (x *DtmRequest) Reset() {
@@ -212,6 +206,20 @@ func (x *DtmRequest) GetSteps() string {
 	return ""
 }
 
+func (x *DtmRequest) GetReqExtra() map[string]string {
+	if x != nil {
+		return x.ReqExtra
+	}
+	return nil
+}
+
+func (x *DtmRequest) GetRollbackReason() string {
+	if x != nil {
+		return x.RollbackReason
+	}
+	return ""
+}
+
 type DtmGidReply struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -346,6 +354,124 @@ func (x *DtmBranchRequest) GetBusiPayload() []byte {
 	return nil
 }
 
+type DtmProgressesReply struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Progresses []*DtmProgress `protobuf:"bytes,1,rep,name=Progresses,proto3" json:"Progresses,omitempty"`
+}
+
+func (x *DtmProgressesReply) Reset() {
+	*x = DtmProgressesReply{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *DtmProgressesReply) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DtmProgressesReply) ProtoMessage() {}
+
+func (x *DtmProgressesReply) ProtoReflect() protoreflect.Message {
+	mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DtmProgressesReply.ProtoReflect.Descriptor instead.
+func (*DtmProgressesReply) Descriptor() ([]byte, []int) {
+	return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *DtmProgressesReply) GetProgresses() []*DtmProgress {
+	if x != nil {
+		return x.Progresses
+	}
+	return nil
+}
+
+type DtmProgress struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Status   string `protobuf:"bytes,1,opt,name=Status,proto3" json:"Status,omitempty"`
+	BinData  []byte `protobuf:"bytes,2,opt,name=BinData,proto3" json:"BinData,omitempty"`
+	BranchID string `protobuf:"bytes,3,opt,name=BranchID,proto3" json:"BranchID,omitempty"`
+	Op       string `protobuf:"bytes,4,opt,name=Op,proto3" json:"Op,omitempty"`
+}
+
+func (x *DtmProgress) Reset() {
+	*x = DtmProgress{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *DtmProgress) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DtmProgress) ProtoMessage() {}
+
+func (x *DtmProgress) ProtoReflect() protoreflect.Message {
+	mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DtmProgress.ProtoReflect.Descriptor instead.
+func (*DtmProgress) Descriptor() ([]byte, []int) {
+	return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *DtmProgress) GetStatus() string {
+	if x != nil {
+		return x.Status
+	}
+	return ""
+}
+
+func (x *DtmProgress) GetBinData() []byte {
+	if x != nil {
+		return x.BinData
+	}
+	return nil
+}
+
+func (x *DtmProgress) GetBranchID() string {
+	if x != nil {
+		return x.BranchID
+	}
+	return ""
+}
+
+func (x *DtmProgress) GetOp() string {
+	if x != nil {
+		return x.Op
+	}
+	return ""
+}
+
 var File_dtmgrpc_dtmgpb_dtmgimp_proto protoreflect.FileDescriptor
 
 var file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc = []byte{
@@ -353,7 +479,7 @@ var file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc = []byte{
 	0x2f, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07,
 	0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f,
 	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x22, 0x92, 0x03, 0x0a, 0x0f, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e,
+	0x72, 0x6f, 0x74, 0x6f, 0x22, 0xea, 0x02, 0x0a, 0x0f, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e,
 	0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x57, 0x61, 0x69, 0x74,
 	0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x57, 0x61,
 	0x69, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x69, 0x6d, 0x65,
@@ -371,69 +497,93 @@ var file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc = []byte{
 	0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68,
 	0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65,
 	0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52,
-	0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12,
+	0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x1a,
+	0x40, 0x0a, 0x12, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73,
+	0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38,
+	0x01, 0x22, 0xa0, 0x03, 0x0a, 0x0a, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47,
+	0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65,
+	0x12, 0x3c, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70,
+	0x2e, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x52, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22,
+	0x0a, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x18, 0x04,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x65, 0x64, 0x44, 0x61,
+	0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x42, 0x69, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, 0x42, 0x69, 0x6e, 0x50, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x50, 0x72, 0x65,
+	0x70, 0x61, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x51, 0x75, 0x65,
+	0x72, 0x79, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x74,
+	0x65, 0x70, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x53, 0x74, 0x65, 0x70, 0x73,
+	0x12, 0x3d, 0x0a, 0x08, 0x52, 0x65, 0x71, 0x45, 0x78, 0x74, 0x72, 0x61, 0x18, 0x08, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x21, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x71, 0x45, 0x78, 0x74, 0x72, 0x61,
+	0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x52, 0x65, 0x71, 0x45, 0x78, 0x74, 0x72, 0x61, 0x12,
 	0x26, 0x0a, 0x0e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x73, 0x6f,
-	0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63,
+	0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63,
-	0x6b, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x1a, 0x40, 0x0a, 0x12, 0x42, 0x72, 0x61, 0x6e, 0x63,
+	0x6b, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x1a, 0x3b, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x45, 0x78,
-	0x68, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+	0x74, 0x72, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
-	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
-	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
+	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
-	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfc, 0x01, 0x0a, 0x0a, 0x44, 0x74,
+	0x3a, 0x02, 0x38, 0x01, 0x22, 0x1f, 0x0a, 0x0b, 0x44, 0x74, 0x6d, 0x47, 0x69, 0x64, 0x52, 0x65,
-	0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18,
+	0x70, 0x6c, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x72,
+	0x52, 0x03, 0x47, 0x69, 0x64, 0x22, 0x82, 0x02, 0x0a, 0x10, 0x44, 0x74, 0x6d, 0x42, 0x72, 0x61,
-	0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x54,
+	0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69,
-	0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e,
+	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09,
-	0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18,
+	0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e,
+	0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x42, 0x72,
-	0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f,
+	0x61, 0x6e, 0x63, 0x68, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x42, 0x72,
-	0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d,
+	0x61, 0x6e, 0x63, 0x68, 0x49, 0x44, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x70, 0x18, 0x04, 0x20, 0x01,
-	0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x75,
+	0x28, 0x09, 0x52, 0x02, 0x4f, 0x70, 0x12, 0x37, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x18, 0x05,
-	0x73, 0x74, 0x6f, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x42, 0x69,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44,
-	0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52,
+	0x74, 0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e,
-	0x0b, 0x42, 0x69, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x24, 0x0a, 0x0d,
+	0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12,
-	0x51, 0x75, 0x65, 0x72, 0x79, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20,
+	0x20, 0x0a, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x06,
-	0x01, 0x28, 0x09, 0x52, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72,
+	0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
-	0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28,
+	0x64, 0x1a, 0x37, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x09, 0x52, 0x05, 0x53, 0x74, 0x65, 0x70, 0x73, 0x22, 0x1f, 0x0a, 0x0b, 0x44, 0x74, 0x6d, 0x47,
+	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x69, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01,
+	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64, 0x22, 0x82, 0x02, 0x0a, 0x10, 0x44, 0x74,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4a, 0x0a, 0x12, 0x44, 0x74,
-	0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10,
+	0x6d, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79,
-	0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64,
+	0x12, 0x34, 0x0a, 0x0a, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01,
-	0x12, 0x1c, 0x0a, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44,
-	0x01, 0x28, 0x09, 0x52, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a,
+	0x74, 0x6d, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x0a, 0x50, 0x72, 0x6f, 0x67,
-	0x0a, 0x08, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
+	0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x6b, 0x0a, 0x0b, 0x44, 0x74, 0x6d, 0x50, 0x72, 0x6f,
-	0x52, 0x08, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x49, 0x44, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x70,
+	0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18,
-	0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x4f, 0x70, 0x12, 0x37, 0x0a, 0x04, 0x44, 0x61,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a,
-	0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69,
+	0x07, 0x42, 0x69, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07,
-	0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75,
+	0x42, 0x69, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x42, 0x72, 0x61, 0x6e, 0x63,
-	0x65, 0x73, 0x74, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x44,
+	0x68, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x42, 0x72, 0x61, 0x6e, 0x63,
-	0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61, 0x79, 0x6c, 0x6f,
+	0x68, 0x49, 0x44, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x61, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61,
+	0x02, 0x4f, 0x70, 0x32, 0xf8, 0x02, 0x0a, 0x03, 0x44, 0x74, 0x6d, 0x12, 0x38, 0x0a, 0x06, 0x4e,
-	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x1a, 0x37, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74,
+	0x65, 0x77, 0x47, 0x69, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
-	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x14, 0x2e,
-	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
+	0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x47, 0x69, 0x64, 0x52, 0x65,
-	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xb1,
+	0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x06, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x12,
-	0x02, 0x0a, 0x03, 0x44, 0x74, 0x6d, 0x12, 0x38, 0x0a, 0x06, 0x4e, 0x65, 0x77, 0x47, 0x69, 0x64,
+	0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71,
-	0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
-	0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x14, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x38,
-	0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x47, 0x69, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00,
+	0x0a, 0x07, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67,
-	0x12, 0x37, 0x0a, 0x06, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d,
+	0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16,
+	0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x05, 0x41, 0x62, 0x6f, 0x72,
+	0x74, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00,
+	0x12, 0x45, 0x0a, 0x0e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x72, 0x61, 0x6e,
+	0x63, 0x68, 0x12, 0x19, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d,
+	0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
+	0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0f, 0x50, 0x72, 0x65, 0x70, 0x61,
+	0x72, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d,
 	0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
+	0x1b, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x50, 0x72, 0x6f,
-	0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x07, 0x50, 0x72, 0x65,
+	0x67, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a,
-	0x70, 0x61, 0x72, 0x65, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44,
+	0x5a, 0x08, 0x2e, 0x2f, 0x64, 0x74, 0x6d, 0x67, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
-	0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x6f, 0x33,
-	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
-	0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x05, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x12, 0x13, 0x2e, 0x64,
-	0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0e, 0x52,
-	0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x12, 0x19, 0x2e,
-	0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63,
-	0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
-	0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
-	0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x2f, 0x64, 0x74, 0x6d, 0x67, 0x70, 0x62, 0x62, 0x06,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -448,35 +598,42 @@ func file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP() []byte {
 	return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescData
 }
 
-var file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
+var file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
 var file_dtmgrpc_dtmgpb_dtmgimp_proto_goTypes = []interface{}{
-	(*DtmTransOptions)(nil),  // 0: dtmgimp.DtmTransOptions
+	(*DtmTransOptions)(nil),    // 0: dtmgimp.DtmTransOptions
-	(*DtmRequest)(nil),       // 1: dtmgimp.DtmRequest
+	(*DtmRequest)(nil),         // 1: dtmgimp.DtmRequest
-	(*DtmGidReply)(nil),      // 2: dtmgimp.DtmGidReply
+	(*DtmGidReply)(nil),        // 2: dtmgimp.DtmGidReply
-	(*DtmBranchRequest)(nil), // 3: dtmgimp.DtmBranchRequest
+	(*DtmBranchRequest)(nil),   // 3: dtmgimp.DtmBranchRequest
-	nil,                      // 4: dtmgimp.DtmTransOptions.BranchHeadersEntry
+	(*DtmProgressesReply)(nil), // 4: dtmgimp.DtmProgressesReply
-	nil,                      // 5: dtmgimp.DtmBranchRequest.DataEntry
+	(*DtmProgress)(nil),        // 5: dtmgimp.DtmProgress
-	(*emptypb.Empty)(nil),    // 6: google.protobuf.Empty
+	nil,                        // 6: dtmgimp.DtmTransOptions.BranchHeadersEntry
+	nil,                        // 7: dtmgimp.DtmRequest.ReqExtraEntry
+	nil,                        // 8: dtmgimp.DtmBranchRequest.DataEntry
+	(*emptypb.Empty)(nil),      // 9: google.protobuf.Empty
 }
 var file_dtmgrpc_dtmgpb_dtmgimp_proto_depIdxs = []int32{
-	4, // 0: dtmgimp.DtmTransOptions.BranchHeaders:type_name -> dtmgimp.DtmTransOptions.BranchHeadersEntry
+	6,  // 0: dtmgimp.DtmTransOptions.BranchHeaders:type_name -> dtmgimp.DtmTransOptions.BranchHeadersEntry
-	0, // 1: dtmgimp.DtmRequest.TransOptions:type_name -> dtmgimp.DtmTransOptions
+	0,  // 1: dtmgimp.DtmRequest.TransOptions:type_name -> dtmgimp.DtmTransOptions
-	5, // 2: dtmgimp.DtmBranchRequest.Data:type_name -> dtmgimp.DtmBranchRequest.DataEntry
+	7,  // 2: dtmgimp.DtmRequest.ReqExtra:type_name -> dtmgimp.DtmRequest.ReqExtraEntry
-	6, // 3: dtmgimp.Dtm.NewGid:input_type -> google.protobuf.Empty
+	8,  // 3: dtmgimp.DtmBranchRequest.Data:type_name -> dtmgimp.DtmBranchRequest.DataEntry
-	1, // 4: dtmgimp.Dtm.Submit:input_type -> dtmgimp.DtmRequest
+	5,  // 4: dtmgimp.DtmProgressesReply.Progresses:type_name -> dtmgimp.DtmProgress
-	1, // 5: dtmgimp.Dtm.Prepare:input_type -> dtmgimp.DtmRequest
+	9,  // 5: dtmgimp.Dtm.NewGid:input_type -> google.protobuf.Empty
-	1, // 6: dtmgimp.Dtm.Abort:input_type -> dtmgimp.DtmRequest
+	1,  // 6: dtmgimp.Dtm.Submit:input_type -> dtmgimp.DtmRequest
-	3, // 7: dtmgimp.Dtm.RegisterBranch:input_type -> dtmgimp.DtmBranchRequest
+	1,  // 7: dtmgimp.Dtm.Prepare:input_type -> dtmgimp.DtmRequest
-	2, // 8: dtmgimp.Dtm.NewGid:output_type -> dtmgimp.DtmGidReply
+	1,  // 8: dtmgimp.Dtm.Abort:input_type -> dtmgimp.DtmRequest
-	6, // 9: dtmgimp.Dtm.Submit:output_type -> google.protobuf.Empty
+	3,  // 9: dtmgimp.Dtm.RegisterBranch:input_type -> dtmgimp.DtmBranchRequest
-	6, // 10: dtmgimp.Dtm.Prepare:output_type -> google.protobuf.Empty
+	1,  // 10: dtmgimp.Dtm.PrepareWorkflow:input_type -> dtmgimp.DtmRequest
-	6, // 11: dtmgimp.Dtm.Abort:output_type -> google.protobuf.Empty
+	2,  // 11: dtmgimp.Dtm.NewGid:output_type -> dtmgimp.DtmGidReply
-	6, // 12: dtmgimp.Dtm.RegisterBranch:output_type -> google.protobuf.Empty
+	9,  // 12: dtmgimp.Dtm.Submit:output_type -> google.protobuf.Empty
-	8, // [8:13] is the sub-list for method output_type
+	9,  // 13: dtmgimp.Dtm.Prepare:output_type -> google.protobuf.Empty
-	3, // [3:8] is the sub-list for method input_type
+	9,  // 14: dtmgimp.Dtm.Abort:output_type -> google.protobuf.Empty
-	3, // [3:3] is the sub-list for extension type_name
+	9,  // 15: dtmgimp.Dtm.RegisterBranch:output_type -> google.protobuf.Empty
-	3, // [3:3] is the sub-list for extension extendee
+	4,  // 16: dtmgimp.Dtm.PrepareWorkflow:output_type -> dtmgimp.DtmProgressesReply
-	0, // [0:3] is the sub-list for field type_name
+	11, // [11:17] is the sub-list for method output_type
+	5,  // [5:11] is the sub-list for method input_type
+	5,  // [5:5] is the sub-list for extension type_name
+	5,  // [5:5] is the sub-list for extension extendee
+	0,  // [0:5] is the sub-list for field type_name
 }
 
 func init() { file_dtmgrpc_dtmgpb_dtmgimp_proto_init() }
@@ -533,6 +690,30 @@ func file_dtmgrpc_dtmgpb_dtmgimp_proto_init() {
 				return nil
 			}
 		}
+		file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DtmProgressesReply); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DtmProgress); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -540,7 +721,7 @@ func file_dtmgrpc_dtmgpb_dtmgimp_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   6,
+			NumMessages:   9,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

dtmgrpc/dtmgpb/dtmgimp.proto

@@ -12,6 +12,7 @@ service Dtm {
   rpc Prepare(DtmRequest) returns (google.protobuf.Empty) {}
   rpc Abort(DtmRequest) returns (google.protobuf.Empty) {}
   rpc RegisterBranch(DtmBranchRequest) returns (google.protobuf.Empty) {}
+  rpc PrepareWorkflow(DtmRequest) returns (DtmProgressesReply) {}
 }
 
 message DtmTransOptions {
@@ -21,7 +22,6 @@ message DtmTransOptions {
   repeated string PassthroughHeaders = 4;
   map<string, string> BranchHeaders = 5;
   int64 RequestTimeout = 6;
-  string RollbackReason = 7;
 }
 
 // DtmRequest request sent to dtm server
@@ -30,9 +30,11 @@ message DtmRequest {
   string TransType = 2;
   DtmTransOptions TransOptions = 3;
   string CustomedData = 4;
-  repeated bytes BinPayloads = 5; // for MSG/SAGA branch payloads
+  repeated bytes BinPayloads = 5; // for Msg/Saga/Workflow branch payloads
-  string QueryPrepared = 6; // for MSG
+  string QueryPrepared = 6; // for Msg
   string Steps = 7;
+  map<string, string> ReqExtra = 8;
+  string RollbackReason = 9;
 }
 
 message DtmGidReply {
@@ -48,3 +50,13 @@ message DtmBranchRequest {
   bytes BusiPayload = 6;
 }
 
+message DtmProgressesReply {
+  repeated DtmProgress Progresses = 1;
+}
+
+message DtmProgress {
+  string Status = 1;
+  bytes BinData = 2;
+  string BranchID = 3;
+  string Op = 4;
+}

dtmgrpc/dtmgpb/dtmgimp_grpc.pb.go

@@ -28,6 +28,7 @@ type DtmClient interface {
 	Prepare(ctx context.Context, in *DtmRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
 	Abort(ctx context.Context, in *DtmRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
 	RegisterBranch(ctx context.Context, in *DtmBranchRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
+	PrepareWorkflow(ctx context.Context, in *DtmRequest, opts ...grpc.CallOption) (*DtmProgressesReply, error)
 }
 
 type dtmClient struct {
@@ -83,6 +84,15 @@ func (c *dtmClient) RegisterBranch(ctx context.Context, in *DtmBranchRequest, op
 	return out, nil
 }
 
+func (c *dtmClient) PrepareWorkflow(ctx context.Context, in *DtmRequest, opts ...grpc.CallOption) (*DtmProgressesReply, error) {
+	out := new(DtmProgressesReply)
+	err := c.cc.Invoke(ctx, "/dtmgimp.Dtm/PrepareWorkflow", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // DtmServer is the server API for Dtm service.
 // All implementations must embed UnimplementedDtmServer
 // for forward compatibility
@@ -92,6 +102,7 @@ type DtmServer interface {
 	Prepare(context.Context, *DtmRequest) (*emptypb.Empty, error)
 	Abort(context.Context, *DtmRequest) (*emptypb.Empty, error)
 	RegisterBranch(context.Context, *DtmBranchRequest) (*emptypb.Empty, error)
+	PrepareWorkflow(context.Context, *DtmRequest) (*DtmProgressesReply, error)
 	mustEmbedUnimplementedDtmServer()
 }
 
@@ -114,6 +125,9 @@ func (UnimplementedDtmServer) Abort(context.Context, *DtmRequest) (*emptypb.Empt
 func (UnimplementedDtmServer) RegisterBranch(context.Context, *DtmBranchRequest) (*emptypb.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RegisterBranch not implemented")
 }
+func (UnimplementedDtmServer) PrepareWorkflow(context.Context, *DtmRequest) (*DtmProgressesReply, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method PrepareWorkflow not implemented")
+}
 func (UnimplementedDtmServer) mustEmbedUnimplementedDtmServer() {}
 
 // UnsafeDtmServer may be embedded to opt out of forward compatibility for this service.
@@ -217,6 +231,24 @@ func _Dtm_RegisterBranch_Handler(srv interface{}, ctx context.Context, dec func(
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Dtm_PrepareWorkflow_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DtmRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(DtmServer).PrepareWorkflow(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/dtmgimp.Dtm/PrepareWorkflow",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(DtmServer).PrepareWorkflow(ctx, req.(*DtmRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // Dtm_ServiceDesc is the grpc.ServiceDesc for Dtm service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -244,6 +276,10 @@ var Dtm_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "RegisterBranch",
 			Handler:    _Dtm_RegisterBranch_Handler,
 		},
+		{
+			MethodName: "PrepareWorkflow",
+			Handler:    _Dtm_PrepareWorkflow_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "dtmgrpc/dtmgpb/dtmgimp.proto",

dtmgrpc/type.go

@@ -27,23 +27,22 @@ func DtmError2GrpcError(res interface{}) error {
 	if ok && errors.Is(e, dtmimp.ErrFailure) {
 		return status.New(codes.Aborted, e.Error()).Err()
 	} else if ok && errors.Is(e, dtmimp.ErrOngoing) {
-		return status.New(codes.FailedPrecondition, dtmcli.ResultOngoing).Err()
+		return status.New(codes.FailedPrecondition, e.Error()).Err()
 	}
 	return e
 }
 
 // GrpcError2DtmError translate grpc error to dtm error
 func GrpcError2DtmError(err error) error {
-	st, ok := status.FromError(err)
+	st, _ := status.FromError(err)
-	if ok && st.Code() == codes.Aborted {
+	if st != nil && st.Code() == codes.Aborted {
 		// version lower then v1.10, will specify Ongoing in code Aborted
 		if st.Message() == dtmcli.ResultOngoing {
 			return dtmcli.ErrOngoing
 		}
-
 		return fmt.Errorf("%s. %w", st.Message(), dtmcli.ErrFailure)
-	} else if ok && st.Code() == codes.FailedPrecondition {
+	} else if st != nil && st.Code() == codes.FailedPrecondition {
-		return dtmcli.ErrOngoing
+		return fmt.Errorf("%s. %w", st.Message(), dtmcli.ErrOngoing)
 	}
 	return err
 }

dtmgrpc/workflow/dummyReadCloser.go

@@ -0,0 +1,25 @@
+package workflow
+
+import (
+	"bytes"
+	"io"
+)
+
+// NewRespBodyFromBytes creates an io.ReadCloser from a byte slice
+// that is suitable for use as an http response body.
+func NewRespBodyFromBytes(body []byte) io.ReadCloser {
+	return &dummyReadCloser{body: bytes.NewReader(body)}
+}
+
+type dummyReadCloser struct {
+	body io.ReadSeeker
+}
+
+func (d *dummyReadCloser) Read(p []byte) (n int, err error) {
+	return d.body.Read(p)
+}
+
+func (d *dummyReadCloser) Close() error {
+	_, _ = d.body.Seek(0, io.SeekEnd)
+	return nil
+}

dtmgrpc/workflow/factory.go

@@ -0,0 +1,52 @@
+package workflow
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/dtm-labs/dtm/dtmcli/logger"
+)
+
+type workflowFactory struct {
+	protocol     string
+	httpDtm      string
+	httpCallback string
+	grpcDtm      string
+	grpcCallback string
+	handlers     map[string]*wfItem
+}
+
+var defaultFac = workflowFactory{
+	handlers: map[string]*wfItem{},
+}
+
+func (w *workflowFactory) execute(name string, gid string, data []byte) error {
+	handler := w.handlers[name]
+	if handler == nil {
+		return fmt.Errorf("workflow '%s' not registered. please register at startup", name)
+	}
+	wf := w.newWorkflow(name, gid, data)
+	for _, fn := range handler.custom {
+		fn(wf)
+	}
+	return wf.process(handler.fn, data)
+}
+
+func (w *workflowFactory) executeByQS(qs url.Values, body []byte) error {
+	name := qs.Get("op")
+	gid := qs.Get("gid")
+	return w.execute(name, gid, body)
+}
+
+func (w *workflowFactory) register(name string, handler WfFunc, custom ...func(wf *Workflow)) error {
+	e := w.handlers[name]
+	if e != nil {
+		return fmt.Errorf("a handler already exists for %s", name)
+	}
+	logger.Debugf("workflow '%s' registered.", name)
+	w.handlers[name] = &wfItem{
+		fn:     handler,
+		custom: custom,
+	}
+	return nil
+}

dtmgrpc/workflow/imp.go

@@ -0,0 +1,200 @@
+package workflow
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmcli/logger"
+	"github.com/dtm-labs/dtm/dtmgrpc"
+	"github.com/go-resty/resty/v2"
+)
+
+type workflowImp struct {
+	restyClient          *resty.Client //nolint
+	idGen                dtmimp.BranchIDGen
+	currentBranch        string                 //nolint
+	currentActionAdded   bool                   //nolint
+	currentCommitAdded   bool                   //nolint
+	currentRollbackAdded bool                   //nolint
+	currentRollbackItem  *workflowPhase2Item    // nolint
+	progresses           map[string]*stepResult //nolint
+	currentOp            string
+	succeededOps         []workflowPhase2Item
+	failedOps            []workflowPhase2Item
+}
+
+type workflowPhase2Item struct {
+	branchID, op string
+	fn           WfPhase2Func
+}
+
+func (wf *Workflow) loadProgresses() error {
+	progresses, err := wf.getProgress()
+	if err == nil {
+		wf.progresses = map[string]*stepResult{}
+		for _, p := range progresses {
+			sr := &stepResult{
+				Status: p.Status,
+				Data:   p.BinData,
+			}
+			if sr.Status == dtmcli.StatusFailed {
+				sr.Error = fmt.Errorf("%s. %w", string(p.BinData), dtmcli.ErrFailure)
+			}
+			wf.progresses[p.BranchID+"-"+p.Op] = sr
+		}
+	}
+	return err
+}
+
+type wfMeta struct{}
+
+func (w *workflowFactory) newWorkflow(name string, gid string, data []byte) *Workflow {
+	wf := &Workflow{
+		TransBase: dtmimp.NewTransBase(gid, "workflow", "not inited", ""),
+		Name:      name,
+		workflowImp: workflowImp{
+			idGen:        dtmimp.BranchIDGen{},
+			succeededOps: []workflowPhase2Item{},
+			failedOps:    []workflowPhase2Item{},
+			currentOp:    dtmimp.OpAction,
+		},
+	}
+	wf.Protocol = w.protocol
+	if w.protocol == dtmimp.ProtocolGRPC {
+		wf.Dtm = w.grpcDtm
+		wf.QueryPrepared = w.grpcCallback
+	} else {
+		wf.Dtm = w.httpDtm
+		wf.QueryPrepared = w.httpCallback
+	}
+	wf.CustomData = dtmimp.MustMarshalString(map[string]interface{}{
+		"name": wf.Name,
+		"data": data,
+	})
+	wf.Context = context.WithValue(wf.Context, wfMeta{}, wf)
+	wf.Options.HTTPResp2DtmError = HTTPResp2DtmError
+	wf.Options.GRPCError2DtmError = dtmgrpc.GrpcError2DtmError
+	wf.initRestyClient()
+	return wf
+}
+
+func (wf *Workflow) initRestyClient() {
+	wf.restyClient = resty.New()
+	wf.restyClient.OnBeforeRequest(func(c *resty.Client, r *resty.Request) error {
+		r.SetQueryParams(map[string]string{
+			"gid":        wf.Gid,
+			"trans_type": wf.TransType,
+			"branch_id":  wf.currentBranch,
+			"op":         dtmimp.OpAction,
+		})
+		err := dtmimp.BeforeRequest(c, r)
+		return err
+	})
+	old := wf.restyClient.GetClient().Transport
+	wf.restyClient.GetClient().Transport = newRoundTripper(old, wf)
+	wf.restyClient.OnAfterResponse(func(c *resty.Client, r *resty.Response) error {
+		return dtmimp.AfterResponse(c, r)
+	})
+}
+
+func (wf *Workflow) process(handler WfFunc, data []byte) (err error) {
+	err = wf.loadProgresses()
+	if err == nil {
+		err = handler(wf, data)
+		err = dtmgrpc.GrpcError2DtmError(err)
+		if err != nil && !errors.Is(err, dtmcli.ErrFailure) {
+			return err
+		}
+		err = wf.processPhase2(err)
+	}
+	if err == nil || errors.Is(err, dtmcli.ErrFailure) {
+		err1 := wf.submit(wfErrorToStatus(err))
+		if err1 != nil {
+			return err1
+		}
+	}
+	return err
+
+}
+
+func (wf *Workflow) saveResult(branchID string, op string, sr *stepResult) error {
+	if sr.Status != "" {
+		err := wf.registerBranch(sr.Data, branchID, op, sr.Status)
+		if err != nil {
+			return err
+		}
+	}
+	return sr.Error
+}
+
+func (wf *Workflow) processPhase2(err error) error {
+	ops := wf.succeededOps
+	if err == nil {
+		wf.currentOp = dtmimp.OpCommit
+	} else {
+		wf.currentOp = dtmimp.OpRollback
+		ops = wf.failedOps
+	}
+	for i := len(ops) - 1; i >= 0; i-- {
+		op := ops[i]
+
+		err1 := wf.callPhase2(op.branchID, op.fn)
+		if err1 != nil {
+			return err1
+		}
+	}
+	return err
+}
+
+func (wf *Workflow) callPhase2(branchID string, fn WfPhase2Func) error {
+	wf.currentBranch = branchID
+	r := wf.recordedDo(func(bb *dtmcli.BranchBarrier) *stepResult {
+		err := fn(bb)
+		dtmimp.PanicIf(errors.Is(err, dtmcli.ErrFailure), errors.New("should not return ErrFail in phase2"))
+		return wf.stepResultFromLocal(nil, err)
+	})
+	_, err := wf.stepResultToLocal(r)
+	return err
+}
+
+func (wf *Workflow) recordedDo(fn func(bb *dtmcli.BranchBarrier) *stepResult) *stepResult {
+	sr := wf.recordedDoInner(fn)
+	if wf.currentRollbackItem != nil && (sr.Status == dtmcli.StatusSucceed || sr.Status == dtmcli.StatusFailed && wf.Options.CompensateErrorBranch) {
+		wf.failedOps = append(wf.failedOps, *wf.currentRollbackItem)
+	}
+	wf.currentRollbackItem = nil
+	return sr
+}
+
+func (wf *Workflow) recordedDoInner(fn func(bb *dtmcli.BranchBarrier) *stepResult) *stepResult {
+	branchID := wf.currentBranch
+	if wf.currentOp == dtmimp.OpAction {
+		dtmimp.PanicIf(wf.currentActionAdded, fmt.Errorf("one branch can have only on action"))
+		wf.currentActionAdded = true
+	}
+	r := wf.getStepResult()
+	if r != nil {
+		logger.Debugf("progress restored: %s %s %v %s %s", branchID, wf.currentOp, r.Error, r.Status, r.Data)
+		return r
+	}
+	bb := &dtmcli.BranchBarrier{
+		TransType: wf.TransType,
+		Gid:       wf.Gid,
+		BranchID:  branchID,
+		Op:        wf.currentOp,
+	}
+	r = fn(bb)
+	err := wf.saveResult(branchID, wf.currentOp, r)
+	if err != nil {
+		r = wf.stepResultFromLocal(nil, err)
+	}
+	return r
+}
+
+func (wf *Workflow) getStepResult() *stepResult {
+	logger.Debugf("getStepResult: %s %v", wf.currentBranch+"-"+wf.currentOp, wf.progresses[wf.currentBranch+"-"+wf.currentOp])
+	return wf.progresses[wf.currentBranch+"-"+wf.currentOp]
+}

dtmgrpc/workflow/rpc.go

@@ -0,0 +1,67 @@
+package workflow
+
+import (
+	"context"
+
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+func (wf *Workflow) getProgress() ([]*dtmgpb.DtmProgress, error) {
+	if wf.Protocol == dtmimp.ProtocolGRPC {
+		var reply dtmgpb.DtmProgressesReply
+		err := dtmgimp.MustGetGrpcConn(wf.Dtm, false).Invoke(wf.Context, "/dtmgimp.Dtm/PrepareWorkflow",
+			dtmgimp.GetDtmRequest(wf.TransBase), &reply)
+		if err == nil {
+			return reply.Progresses, nil
+		}
+		return nil, err
+	}
+	resp, err := dtmimp.RestyClient.R().SetBody(wf.TransBase).Post(wf.Dtm + "/prepareWorkflow")
+	var progresses []*dtmgpb.DtmProgress
+	if err == nil {
+		dtmimp.MustUnmarshal(resp.Body(), &progresses)
+	}
+	return progresses, err
+}
+
+func (wf *Workflow) submit(status string) error {
+	if wf.Protocol == dtmimp.ProtocolHTTP {
+		m := map[string]interface{}{
+			"gid":        wf.Gid,
+			"trans_type": wf.TransType,
+			"req_extra": map[string]string{
+				"status": status,
+			},
+		}
+		_, err := dtmimp.TransCallDtmExt(wf.TransBase, m, "submit")
+		return err
+	}
+	req := dtmgimp.GetDtmRequest(wf.TransBase)
+	req.ReqExtra = map[string]string{
+		"status": status,
+	}
+	reply := emptypb.Empty{}
+	return dtmgimp.MustGetGrpcConn(wf.Dtm, false).Invoke(wf.Context, "/dtmgimp.Dtm/"+"Submit", req, &reply)
+}
+
+func (wf *Workflow) registerBranch(res []byte, branchID string, op string, status string) error {
+	if wf.Protocol == dtmimp.ProtocolHTTP {
+		return dtmimp.TransRegisterBranch(wf.TransBase, map[string]string{
+			"data":      string(res),
+			"branch_id": branchID,
+			"op":        op,
+			"status":    status,
+		}, "registerBranch")
+	}
+	_, err := dtmgimp.MustGetDtmClient(wf.Dtm).RegisterBranch(context.Background(), &dtmgpb.DtmBranchRequest{
+		Gid:         wf.Gid,
+		TransType:   wf.TransType,
+		BranchID:    branchID,
+		BusiPayload: res,
+		Data:        map[string]string{"status": status, "op": op},
+	})
+	return err
+}

dtmgrpc/workflow/server.go

@@ -0,0 +1,26 @@
+package workflow
+
+import (
+	"context"
+
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmgrpc"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow/wfpb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+type workflowServer struct {
+	wfpb.UnimplementedWorkflowServer
+}
+
+func (s *workflowServer) Execute(ctx context.Context, wd *wfpb.WorkflowData) (*emptypb.Empty, error) {
+	if defaultFac.protocol != dtmimp.ProtocolGRPC {
+		return nil, status.Errorf(codes.Internal, "workflow server not inited. please call workflow.InitGrpc first")
+	}
+	tb := dtmgimp.TransBaseFromGrpc(ctx)
+	err := defaultFac.execute(tb.Op, tb.Gid, wd.Data)
+	return &emptypb.Empty{}, dtmgrpc.DtmError2GrpcError(err)
+}

dtmgrpc/workflow/utils.go

@@ -0,0 +1,129 @@
+package workflow
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+func wfErrorToStatus(err error) string {
+	if err == nil {
+		return dtmcli.StatusSucceed
+	} else if errors.Is(err, dtmcli.ErrFailure) {
+		return dtmcli.StatusFailed
+	}
+	return ""
+}
+
+type stepResult struct {
+	Error  error  // if Error != nil || Status == "", result will not be saved
+	Status string // succeed | failed | ""
+	// if status == succeed, data is the result.
+	// if status == failed, data is the error message
+	Data []byte
+}
+
+type roundTripper struct {
+	old http.RoundTripper
+	wf  *Workflow
+}
+
+func newJSONResponse(status int, result []byte) *http.Response {
+	return &http.Response{
+		Status:     strconv.Itoa(status),
+		StatusCode: status,
+		Body:       NewRespBodyFromBytes(result),
+		Header: http.Header{
+			"Content-Type": []string{"application/json"},
+		},
+		ContentLength: -1,
+	}
+}
+
+func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	wf := r.wf
+	origin := func(bb *dtmcli.BranchBarrier) *stepResult {
+		resp, err := r.old.RoundTrip(req)
+		return wf.stepResultFromHTTP(resp, err)
+	}
+	var sr *stepResult
+	if wf.currentOp != dtmimp.OpAction { // in phase 2, do not save, because it is saved outer
+		sr = origin(nil)
+	} else {
+		sr = wf.recordedDo(origin)
+	}
+	return wf.stepResultToHTTP(sr)
+}
+
+func newRoundTripper(old http.RoundTripper, wf *Workflow) http.RoundTripper {
+	return &roundTripper{old: old, wf: wf}
+}
+
+// HTTPResp2DtmError check for dtm error and return it
+func HTTPResp2DtmError(resp *http.Response) ([]byte, error) {
+	code := resp.StatusCode
+	data, err := ioutil.ReadAll(resp.Body)
+	resp.Body = ioutil.NopCloser(bytes.NewBuffer(data))
+	if code == http.StatusTooEarly {
+		return data, fmt.Errorf("%s. %w", string(data), dtmcli.ErrOngoing)
+	} else if code == http.StatusConflict {
+		return data, fmt.Errorf("%s. %w", string(data), dtmcli.ErrFailure)
+	} else if err == nil && code != http.StatusOK {
+		return data, errors.New(string(data))
+	}
+	return data, err
+}
+
+func (wf *Workflow) stepResultFromLocal(data []byte, err error) *stepResult {
+	return &stepResult{
+		Error:  err,
+		Status: wfErrorToStatus(err),
+		Data:   data,
+	}
+}
+
+func (wf *Workflow) stepResultToLocal(sr *stepResult) ([]byte, error) {
+	return sr.Data, sr.Error
+}
+
+func (wf *Workflow) stepResultFromGrpc(reply interface{}, err error) *stepResult {
+	sr := &stepResult{Error: wf.Options.GRPCError2DtmError(err)}
+	sr.Status = wfErrorToStatus(sr.Error)
+	if sr.Error == nil {
+		sr.Data = dtmgimp.MustProtoMarshal(reply.(protoreflect.ProtoMessage))
+	} else if sr.Status == dtmcli.StatusFailed {
+		sr.Data = []byte(sr.Error.Error())
+	}
+	return sr
+}
+
+func (wf *Workflow) stepResultToGrpc(s *stepResult, reply interface{}) error {
+	if s.Error == nil && s.Status == dtmcli.StatusSucceed {
+		dtmgimp.MustProtoUnmarshal(s.Data, reply.(protoreflect.ProtoMessage))
+	}
+	return s.Error
+}
+
+func (wf *Workflow) stepResultFromHTTP(resp *http.Response, err error) *stepResult {
+	sr := &stepResult{Error: err}
+	if err == nil {
+		sr.Data, sr.Error = wf.Options.HTTPResp2DtmError(resp)
+		sr.Status = wfErrorToStatus(sr.Error)
+	}
+	return sr
+}
+
+func (wf *Workflow) stepResultToHTTP(s *stepResult) (*http.Response, error) {
+	if s.Error != nil {
+		return nil, s.Error
+	}
+	return newJSONResponse(200, s.Data), nil
+}

dtmgrpc/workflow/wfpb/wf.pb.go

@@ -0,0 +1,153 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.0
+// 	protoc        v3.17.3
+// source: dtmgrpc/workflow/wfpb/wf.proto
+
+package wfpb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	emptypb "google.golang.org/protobuf/types/known/emptypb"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type WorkflowData struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Data []byte `protobuf:"bytes,1,opt,name=Data,proto3" json:"Data,omitempty"`
+}
+
+func (x *WorkflowData) Reset() {
+	*x = WorkflowData{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_dtmgrpc_workflow_wfpb_wf_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *WorkflowData) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*WorkflowData) ProtoMessage() {}
+
+func (x *WorkflowData) ProtoReflect() protoreflect.Message {
+	mi := &file_dtmgrpc_workflow_wfpb_wf_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use WorkflowData.ProtoReflect.Descriptor instead.
+func (*WorkflowData) Descriptor() ([]byte, []int) {
+	return file_dtmgrpc_workflow_wfpb_wf_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *WorkflowData) GetData() []byte {
+	if x != nil {
+		return x.Data
+	}
+	return nil
+}
+
+var File_dtmgrpc_workflow_wfpb_wf_proto protoreflect.FileDescriptor
+
+var file_dtmgrpc_workflow_wfpb_wf_proto_rawDesc = []byte{
+	0x0a, 0x1e, 0x64, 0x74, 0x6d, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c,
+	0x6f, 0x77, 0x2f, 0x77, 0x66, 0x70, 0x62, 0x2f, 0x77, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x12, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74,
+	0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x22, 0x0a, 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x66,
+	0x6c, 0x6f, 0x77, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x32, 0x47, 0x0a, 0x08, 0x57,
+	0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x3b, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75,
+	0x74, 0x65, 0x12, 0x16, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x57, 0x6f,
+	0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
+	0x74, 0x79, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x2f, 0x77, 0x66, 0x70, 0x62, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_dtmgrpc_workflow_wfpb_wf_proto_rawDescOnce sync.Once
+	file_dtmgrpc_workflow_wfpb_wf_proto_rawDescData = file_dtmgrpc_workflow_wfpb_wf_proto_rawDesc
+)
+
+func file_dtmgrpc_workflow_wfpb_wf_proto_rawDescGZIP() []byte {
+	file_dtmgrpc_workflow_wfpb_wf_proto_rawDescOnce.Do(func() {
+		file_dtmgrpc_workflow_wfpb_wf_proto_rawDescData = protoimpl.X.CompressGZIP(file_dtmgrpc_workflow_wfpb_wf_proto_rawDescData)
+	})
+	return file_dtmgrpc_workflow_wfpb_wf_proto_rawDescData
+}
+
+var file_dtmgrpc_workflow_wfpb_wf_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_dtmgrpc_workflow_wfpb_wf_proto_goTypes = []interface{}{
+	(*WorkflowData)(nil),  // 0: workflow.WorkflowData
+	(*emptypb.Empty)(nil), // 1: google.protobuf.Empty
+}
+var file_dtmgrpc_workflow_wfpb_wf_proto_depIdxs = []int32{
+	0, // 0: workflow.Workflow.Execute:input_type -> workflow.WorkflowData
+	1, // 1: workflow.Workflow.Execute:output_type -> google.protobuf.Empty
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_dtmgrpc_workflow_wfpb_wf_proto_init() }
+func file_dtmgrpc_workflow_wfpb_wf_proto_init() {
+	if File_dtmgrpc_workflow_wfpb_wf_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_dtmgrpc_workflow_wfpb_wf_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*WorkflowData); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_dtmgrpc_workflow_wfpb_wf_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_dtmgrpc_workflow_wfpb_wf_proto_goTypes,
+		DependencyIndexes: file_dtmgrpc_workflow_wfpb_wf_proto_depIdxs,
+		MessageInfos:      file_dtmgrpc_workflow_wfpb_wf_proto_msgTypes,
+	}.Build()
+	File_dtmgrpc_workflow_wfpb_wf_proto = out.File
+	file_dtmgrpc_workflow_wfpb_wf_proto_rawDesc = nil
+	file_dtmgrpc_workflow_wfpb_wf_proto_goTypes = nil
+	file_dtmgrpc_workflow_wfpb_wf_proto_depIdxs = nil
+}

dtmgrpc/workflow/wfpb/wf.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+option go_package = "./wfpb";
+import "google/protobuf/empty.proto";
+
+package workflow;
+
+// The Workflow service definition.
+service Workflow {
+  rpc Execute(WorkflowData) returns (google.protobuf.Empty) {}
+}
+
+message WorkflowData {
+  bytes Data = 1;
+}

dtmgrpc/workflow/wfpb/wf_grpc.pb.go

@@ -0,0 +1,106 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.17.3
+// source: dtmgrpc/workflow/wfpb/wf.proto
+
+package wfpb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	emptypb "google.golang.org/protobuf/types/known/emptypb"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
+const _ = grpc.SupportPackageIsVersion7
+
+// WorkflowClient is the client API for Workflow service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type WorkflowClient interface {
+	Execute(ctx context.Context, in *WorkflowData, opts ...grpc.CallOption) (*emptypb.Empty, error)
+}
+
+type workflowClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewWorkflowClient(cc grpc.ClientConnInterface) WorkflowClient {
+	return &workflowClient{cc}
+}
+
+func (c *workflowClient) Execute(ctx context.Context, in *WorkflowData, opts ...grpc.CallOption) (*emptypb.Empty, error) {
+	out := new(emptypb.Empty)
+	err := c.cc.Invoke(ctx, "/workflow.Workflow/Execute", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// WorkflowServer is the server API for Workflow service.
+// All implementations must embed UnimplementedWorkflowServer
+// for forward compatibility
+type WorkflowServer interface {
+	Execute(context.Context, *WorkflowData) (*emptypb.Empty, error)
+	mustEmbedUnimplementedWorkflowServer()
+}
+
+// UnimplementedWorkflowServer must be embedded to have forward compatible implementations.
+type UnimplementedWorkflowServer struct {
+}
+
+func (UnimplementedWorkflowServer) Execute(context.Context, *WorkflowData) (*emptypb.Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented")
+}
+func (UnimplementedWorkflowServer) mustEmbedUnimplementedWorkflowServer() {}
+
+// UnsafeWorkflowServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to WorkflowServer will
+// result in compilation errors.
+type UnsafeWorkflowServer interface {
+	mustEmbedUnimplementedWorkflowServer()
+}
+
+func RegisterWorkflowServer(s grpc.ServiceRegistrar, srv WorkflowServer) {
+	s.RegisterService(&Workflow_ServiceDesc, srv)
+}
+
+func _Workflow_Execute_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(WorkflowData)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(WorkflowServer).Execute(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/workflow.Workflow/Execute",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(WorkflowServer).Execute(ctx, req.(*WorkflowData))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// Workflow_ServiceDesc is the grpc.ServiceDesc for Workflow service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var Workflow_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "workflow.Workflow",
+	HandlerType: (*WorkflowServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Execute",
+			Handler:    _Workflow_Execute_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "dtmgrpc/workflow/wfpb/wf.proto",
+}

dtmgrpc/workflow/workflow.go

@@ -0,0 +1,225 @@
+package workflow
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmcli/logger"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow/wfpb"
+	"github.com/go-resty/resty/v2"
+	"google.golang.org/grpc"
+)
+
+// InitHTTP will init Workflow engine to use http
+// param httpDtm specify the dtm address
+// param callback specify the url for dtm to callback if a workflow timeout
+func InitHTTP(httpDtm string, callback string) {
+	defaultFac.protocol = dtmimp.ProtocolHTTP
+	defaultFac.httpDtm = httpDtm
+	defaultFac.httpCallback = callback
+}
+
+// InitGrpc will init Workflow engine to use grpc
+// param dtm specify the dtm address
+// param clientHost specify the client host for dtm to callback if a workflow timeout
+// param grpcServer specify the grpc server
+func InitGrpc(grpcDtm string, clientHost string, grpcServer *grpc.Server) {
+	defaultFac.protocol = dtmimp.ProtocolGRPC
+	defaultFac.grpcDtm = grpcDtm
+	wfpb.RegisterWorkflowServer(grpcServer, &workflowServer{})
+	defaultFac.grpcCallback = clientHost + "/workflow.Workflow/Execute"
+}
+
+// SetProtocolForTest change protocol directly. only used by test
+func SetProtocolForTest(protocol string) {
+	defaultFac.protocol = protocol
+}
+
+// Register will register a workflow with the specified name
+func Register(name string, handler WfFunc, custom ...func(wf *Workflow)) error {
+	return defaultFac.register(name, handler, custom...)
+}
+
+// Execute will execute a workflow with the gid and specified params
+// if the workflow with the gid does not exist, then create a new workflow and execute it
+// if the workflow with the gid exists, resume to execute it
+func Execute(name string, gid string, data []byte) error {
+	return defaultFac.execute(name, gid, data)
+}
+
+// ExecuteByQS is like Execute, but name and gid will be obtained from qs
+func ExecuteByQS(qs url.Values, body []byte) error {
+	return defaultFac.executeByQS(qs, body)
+}
+
+// Options is for specifying workflow options
+type Options struct {
+
+	// Default: Code 409 => ErrFailure; Code 425 => ErrOngoing
+	HTTPResp2DtmError func(*http.Response) ([]byte, error)
+
+	// Default: Code Aborted => ErrFailure; Code FailedPrecondition => ErrOngoing
+	GRPCError2DtmError func(error) error
+
+	// This Option specify whether a branch returning ErrFailure should be compensated on rollback.
+	// for most idempotent branches, no compensation is needed.
+	// But for a timeout request, the caller cannot know where the request is successful, so the compensation should be called
+	CompensateErrorBranch bool
+}
+
+// Workflow is the type for a workflow
+type Workflow struct {
+	// The name of the workflow
+	Name    string
+	Options Options
+	*dtmimp.TransBase
+	workflowImp
+}
+
+type wfItem struct {
+	fn     WfFunc
+	custom []func(*Workflow)
+}
+
+// WfFunc is the type for workflow function
+type WfFunc func(wf *Workflow, data []byte) error
+
+// WfPhase2Func is the type for phase 2 function
+// param bb is a BranchBarrier, which is introduced by http://d.dtm.pub/practice/barrier.html
+type WfPhase2Func func(bb *dtmcli.BranchBarrier) error
+
+// NewRequest return a new resty request, whose progress will be recorded
+func (wf *Workflow) NewRequest() *resty.Request {
+	return wf.restyClient.R().SetContext(wf.Context)
+}
+
+// NewBranch will start a new branch transaction
+func (wf *Workflow) NewBranch() *Workflow {
+	dtmimp.PanicIf(wf.currentOp != dtmimp.OpAction, fmt.Errorf("should not call NewBranch() in Branch callbacks"))
+	wf.idGen.NewSubBranchID()
+	wf.currentBranch = wf.idGen.CurrentSubBranchID()
+	wf.currentActionAdded = false
+	wf.currentCommitAdded = false
+	wf.currentRollbackAdded = false
+	return wf
+}
+
+// NewBranchCtx will call NewBranch and return a workflow context
+func (wf *Workflow) NewBranchCtx() context.Context {
+	return wf.NewBranch().Context
+}
+
+// OnRollback will set the callback for current branch when rollback happen.
+// If you are writing a saga transaction, then you should  write the compensation here
+// If you are writing a tcc transaction, then you should write the cancel operation here
+func (wf *Workflow) OnRollback(compensate WfPhase2Func) *Workflow {
+	branchID := wf.currentBranch
+	dtmimp.PanicIf(wf.currentRollbackAdded, fmt.Errorf("one branch can only add one rollback callback"))
+	wf.currentRollbackAdded = true
+	item := workflowPhase2Item{
+		branchID: branchID,
+		op:       dtmimp.OpRollback,
+		fn:       compensate,
+	}
+	wf.currentRollbackItem = &item
+	return wf
+}
+
+// OnCommit will will set the callback for current branch when commit happen.
+// If you are writing a tcc transaction, then you should write the confirm operation here
+func (wf *Workflow) OnCommit(fn WfPhase2Func) *Workflow {
+	branchID := wf.currentBranch
+	dtmimp.PanicIf(wf.currentCommitAdded, fmt.Errorf("one branch can only add one commit callback"))
+	wf.currentCommitAdded = true
+	wf.failedOps = append(wf.succeededOps, workflowPhase2Item{
+		branchID: branchID,
+		op:       dtmimp.OpCommit,
+		fn:       fn,
+	})
+	return wf
+}
+
+// Do will do an action which will be recored
+func (wf *Workflow) Do(fn func(bb *dtmcli.BranchBarrier) ([]byte, error)) ([]byte, error) {
+	res := wf.recordedDo(func(bb *dtmcli.BranchBarrier) *stepResult {
+		r, e := fn(bb)
+		return wf.stepResultFromLocal(r, e)
+	})
+	return wf.stepResultToLocal(res)
+}
+
+// DoXa will begin a local xa transaction
+// after the return of workflow function, xa commit/rollback will be called
+func (wf *Workflow) DoXa(dbConf dtmcli.DBConf, fn func(db *sql.DB) ([]byte, error)) ([]byte, error) {
+	branchID := wf.currentBranch
+	res := wf.recordedDo(func(bb *dtmcli.BranchBarrier) *stepResult {
+		sBusi := "business"
+		k := bb.BranchID + "-" + sBusi
+		if wf.progresses[k] != nil {
+			return &stepResult{
+				Error: fmt.Errorf("error occur at prepare, not resumable, to rollback. %w", dtmcli.ErrFailure),
+			}
+		}
+		sr := &stepResult{}
+		wf.TransBase.BranchID = branchID
+		wf.TransBase.Op = sBusi
+		err := dtmimp.XaHandleLocalTrans(wf.TransBase, dbConf, func(d *sql.DB) error {
+			r, e := fn(d)
+			sr.Data = r
+			if e == nil {
+				e = wf.saveResult(branchID, sBusi, &stepResult{Status: dtmcli.StatusSucceed})
+			}
+			return e
+		})
+		sr.Error = err
+		sr.Status = wfErrorToStatus(err)
+		return sr
+	})
+	phase2 := func(bb *dtmcli.BranchBarrier) error {
+		return dtmimp.XaHandlePhase2(bb.Gid, dbConf, bb.BranchID, bb.Op)
+	}
+	wf.succeededOps = append(wf.succeededOps, workflowPhase2Item{
+		branchID: branchID,
+		op:       dtmimp.OpCommit,
+		fn:       phase2,
+	})
+	wf.failedOps = append(wf.failedOps, workflowPhase2Item{
+		branchID: branchID,
+		op:       dtmimp.OpRollback,
+		fn:       phase2,
+	})
+	return res.Data, res.Error
+}
+
+// Interceptor is the middleware for workflow to capture grpc call result
+func Interceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+	logger.Debugf("grpc client calling: %s%s %v", cc.Target(), method, dtmimp.MustMarshalString(req))
+	wf := ctx.Value(wfMeta{}).(*Workflow)
+
+	origin := func() error {
+		ctx1 := dtmgimp.TransInfo2Ctx(ctx, wf.Gid, wf.TransType, wf.currentBranch, wf.currentOp, wf.Dtm)
+		err := invoker(ctx1, method, req, reply, cc, opts...)
+		res := fmt.Sprintf("grpc client called: %s%s %s result: %s err: %v",
+			cc.Target(), method, dtmimp.MustMarshalString(req), dtmimp.MustMarshalString(reply), err)
+		if err != nil {
+			logger.Errorf("%s", res)
+		} else {
+			logger.Debugf("%s", res)
+		}
+		return err
+	}
+	if wf.currentOp != dtmimp.OpAction {
+		return origin()
+	}
+	sr := wf.recordedDo(func(bb *dtmcli.BranchBarrier) *stepResult {
+		err := origin()
+		return wf.stepResultFromGrpc(reply, err)
+	})
+	return wf.stepResultToGrpc(sr, reply)
+}

dtmgrpc/workflow/workflow_test.go

@@ -0,0 +1,24 @@
+package workflow
+
+import (
+	"context"
+	"testing"
+
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAbnormal(t *testing.T) {
+	fname := dtmimp.GetFuncName()
+	err := defaultFac.execute(fname, fname, nil)
+	assert.Error(t, err)
+
+	err = defaultFac.register(fname, func(wf *Workflow, data []byte) error { return nil })
+	assert.Nil(t, err)
+	err = defaultFac.register(fname, nil)
+	assert.Error(t, err)
+
+	ws := &workflowServer{}
+	_, err = ws.Execute(context.Background(), nil)
+	assert.Contains(t, err.Error(), "call workflow.InitGrpc first")
+}

dtmsvr/api.go

@@ -20,6 +20,9 @@ var Version = ""
 
 func svcSubmit(t *TransGlobal) interface{} {
 	t.Status = dtmcli.StatusSubmitted
+	if t.ReqExtra != nil && t.ReqExtra["status"] != "" {
+		t.Status = t.ReqExtra["status"]
+	}
 	branches, err := t.saveNew()
 
 	if err == storage.ErrUniqueConflict {
@@ -47,6 +50,15 @@ func svcPrepare(t *TransGlobal) interface{} {
 	return err
 }
 
+func svcPrepareWorkflow(t *TransGlobal) ([]TransBranch, error) {
+	t.Status = dtmcli.StatusPrepared
+	_, err := t.saveNew()
+	if err == storage.ErrUniqueConflict { // transaction exists, query the branches
+		return GetStore().FindBranches(t.Gid), nil
+	}
+	return []TransBranch{}, err
+}
+
 func svcAbort(t *TransGlobal) interface{} {
 	dbt := GetTransGlobal(t.Gid)
 	if dbt.TransType == "msg" && dbt.Status == dtmcli.StatusPrepared {
@@ -82,6 +94,10 @@ func svcRegisterBranch(transType string, branch *TransBranch, data map[string]st
 		branches[0].URL = data["url"]
 		branches[1].Op = dtmimp.OpCommit
 		branches[1].URL = data["url"]
+	} else if transType == "workflow" {
+		branches = []TransBranch{*branch}
+		branches[0].Status = data["status"]
+		branches[0].Op = data["op"]
 	} else {
 		return fmt.Errorf("unknow trans type: %s", transType)
 	}

dtmsvr/api_grpc.go

@@ -48,3 +48,19 @@ func (s *dtmServer) RegisterBranch(ctx context.Context, in *pb.DtmBranchRequest)
 	}, in.Data)
 	return &emptypb.Empty{}, dtmgrpc.DtmError2GrpcError(r)
 }
+
+func (s *dtmServer) PrepareWorkflow(ctx context.Context, in *pb.DtmRequest) (*pb.DtmProgressesReply, error) {
+	branches, err := svcPrepareWorkflow(TransFromDtmRequest(ctx, in))
+	reply := &pb.DtmProgressesReply{
+		Progresses: []*pb.DtmProgress{},
+	}
+	for _, b := range branches {
+		reply.Progresses = append(reply.Progresses, &pb.DtmProgress{
+			Status:   b.Status,
+			BranchID: b.BranchID,
+			Op:       b.Op,
+			BinData:  b.BinData,
+		})
+	}
+	return reply, dtmgrpc.DtmError2GrpcError(err)
+}

dtmsvr/api_http.go

@@ -30,6 +30,7 @@ func addRoute(engine *gin.Engine) {
 	engine.POST("/api/dtmsvr/registerBranch", dtmutil.WrapHandler2(registerBranch))
 	engine.POST("/api/dtmsvr/registerXaBranch", dtmutil.WrapHandler2(registerBranch))  // compatible for old sdk
 	engine.POST("/api/dtmsvr/registerTccBranch", dtmutil.WrapHandler2(registerBranch)) // compatible for old sdk
+	engine.POST("/api/dtmsvr/prepareWorkflow", dtmutil.WrapHandler2(prepareWorkflow))
 	engine.GET("/api/dtmsvr/query", dtmutil.WrapHandler2(query))
 	engine.GET("/api/dtmsvr/all", dtmutil.WrapHandler2(all))
 	engine.GET("/api/dtmsvr/resetCronTime", dtmutil.WrapHandler2(resetCronTime))
@@ -85,6 +86,14 @@ func query(c *gin.Context) interface{} {
 	return map[string]interface{}{"transaction": trans, "branches": branches}
 }
 
+func prepareWorkflow(c *gin.Context) interface{} {
+	branches, err := svcPrepareWorkflow(TransFromContext(c))
+	if err != nil {
+		return err
+	}
+	return branches
+}
+
 func all(c *gin.Context) interface{} {
 	position := c.Query("position")
 	sLimit := dtmimp.OrString(c.Query("limit"), "100")

dtmsvr/cron.go

@@ -35,7 +35,7 @@ func CronTransOnce() (gid string) {
 	trans.WaitResult = true
 	branches := GetStore().FindBranches(gid)
 	err := trans.Process(branches)
-	dtmimp.PanicIf(err != nil && !errors.Is(err, dtmcli.ErrFailure), err)
+	dtmimp.PanicIf(err != nil && !errors.Is(err, dtmcli.ErrFailure) && !errors.Is(err, dtmcli.ErrOngoing), err)
 	return
 }
 

dtmsvr/storage/boltdb/boltdb.go

@@ -69,12 +69,12 @@ func initializeBuckets(db *bolt.DB) error {
 
 // cleanupExpiredData will clean the expired data in boltdb, the
 //    expired time is configurable.
-func cleanupExpiredData(expiredSeconds time.Duration, db *bolt.DB) error {
+func cleanupExpiredData(expire time.Duration, db *bolt.DB) error {
-	if expiredSeconds <= 0 {
+	if expire <= 0 {
 		return nil
 	}
 
-	lastKeepTime := time.Now().Add(-expiredSeconds)
+	lastKeepTime := time.Now().Add(-expire)
 	return db.Update(func(t *bolt.Tx) error {
 		globalBucket := t.Bucket(bucketGlobal)
 		if globalBucket == nil {
@@ -210,8 +210,19 @@ func tPutGlobal(t *bolt.Tx, global *storage.TransGlobalStore) {
 }
 
 func tPutBranches(t *bolt.Tx, branches []storage.TransBranchStore, start int64) {
+	err := tPutBranches2(t, branches, start)
+	dtmimp.E2P(err)
+}
+
+func tPutBranches2(t *bolt.Tx, branches []storage.TransBranchStore, start int64) error {
 	if start == -1 {
-		bs := tGetBranches(t, branches[0].Gid)
+		b0 := &branches[0]
+		bs := tGetBranches(t, b0.Gid)
+		for _, b := range bs {
+			if b.BranchID == b0.BranchID && b.Op == b0.Op {
+				return storage.ErrUniqueConflict
+			}
+		}
 		start = int64(len(bs))
 	}
 	for i, b := range branches {
@@ -220,6 +231,7 @@ func tPutBranches(t *bolt.Tx, branches []storage.TransBranchStore, start int64)
 		err := t.Bucket(bucketBranches).Put([]byte(k), []byte(v))
 		dtmimp.E2P(err)
 	}
+	return nil
 }
 
 func tDelIndex(t *bolt.Tx, unix int64, gid string) {
@@ -323,8 +335,7 @@ func (s *Store) LockGlobalSaveBranches(gid string, status string, branches []sto
 		if g.Status != status {
 			return storage.ErrNotFound
 		}
-		tPutBranches(t, branches, int64(branchStart))
+		return tPutBranches2(t, branches, int64(branchStart))
-		return nil
 	})
 	dtmimp.E2P(err)
 }
@@ -382,34 +393,30 @@ func (s *Store) TouchCronTime(global *storage.TransGlobalStore, nextCronInterval
 
 // LockOneGlobalTrans finds GlobalTrans
 func (s *Store) LockOneGlobalTrans(expireIn time.Duration) *storage.TransGlobalStore {
-	var transo *storage.TransGlobalStore
+	var trans *storage.TransGlobalStore
 	min := fmt.Sprintf("%d", time.Now().Add(expireIn).Unix())
 	err := s.boltDb.Update(func(t *bolt.Tx) error {
 		cursor := t.Bucket(bucketIndex).Cursor()
 		toDelete := [][]byte{}
-		for k, v := cursor.First(); k != nil && string(k) <= min; k, v = cursor.Next() {
+		for k, v := cursor.First(); k != nil && string(k) <= min && (trans == nil || trans.IsFinished()); k, v = cursor.Next() {
+			trans = tGetGlobal(t, string(v))
 			toDelete = append(toDelete, k)
-			trans := tGetGlobal(t, string(v))
-			if trans != nil && !trans.IsFinished() {
-				transo = trans
-				break
-			}
 		}
 		for _, k := range toDelete {
 			err := t.Bucket(bucketIndex).Delete(k)
 			dtmimp.E2P(err)
 		}
-		if transo != nil {
+		if trans != nil && !trans.IsFinished() {
 			next := time.Now().Add(time.Duration(s.retryInterval) * time.Second)
-			transo.NextCronTime = &next
+			trans.NextCronTime = &next
-			tPutGlobal(t, transo)
+			tPutGlobal(t, trans)
-			tPutIndex(t, next.Unix(), transo.Gid)
+			// this put should be after delete, because the data may be the same
-
+			tPutIndex(t, next.Unix(), trans.Gid)
 		}
 		return nil
 	})
 	dtmimp.E2P(err)
-	return transo
+	return trans
 }
 
 // ResetCronTime reset nextCronTime

dtmsvr/storage/redis/redis.go

@@ -198,6 +198,17 @@ if old ~= ARGV[3] then
 	return 'NOT_FOUND'
 end
 local start = ARGV[4]
+-- check duplicates for workflow
+if start == "-1" then
+	local t = cjson.decode(ARGV[5])
+	local bs = redis.call('LRANGE', KEYS[2], 0, -1)
+	for i = 1, table.getn(bs) do
+		local c = cjson.decode(bs[i])
+		if t['branch_id'] == c['branch_id'] and t['op'] == c['op'] then
+			return 'UNIQUE_CONFLICT'
+		end
+	end
+end
 for k = 5, table.getn(ARGV) do
 	if start == "-1" then
 		redis.call('RPUSH', KEYS[2], ARGV[k])

dtmsvr/storage/sql/sql.go

@@ -89,7 +89,11 @@ func (s *Store) LockGlobalSaveBranches(gid string, status string, branches []sto
 		g := &storage.TransGlobalStore{}
 		dbr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Model(g).Where("gid=? and status=?", gid, status).First(g)
 		if dbr.Error == nil {
-			dbr = tx.Save(branches)
+			if branchStart == -1 {
+				dbr = tx.Create(branches)
+			} else {
+				dbr = tx.Save(branches)
+			}
 		}
 		return wrapError(dbr.Error)
 	})

dtmsvr/storage/trans.go

@@ -60,9 +60,9 @@ func (g *TransGlobalStore) IsFinished() bool {
 // TransBranchStore branch transaction
 type TransBranchStore struct {
 	dtmutil.ModelBase
-	Gid          string `json:"gid,omitempty"`
+	Gid          string     `json:"gid,omitempty"`
-	URL          string `json:"url,omitempty"`
+	URL          string     `json:"url,omitempty"`
-	BinData      []byte
+	BinData      []byte     `json:"bin_data,omitempty"`
 	BranchID     string     `json:"branch_id,omitempty"`
 	Op           string     `json:"op,omitempty"`
 	Status       string     `json:"status,omitempty"`

dtmsvr/trans_class.go

@@ -16,6 +16,7 @@ import (
 // TransGlobal global transaction
 type TransGlobal struct {
 	storage.TransGlobalStore
+	ReqExtra         map[string]string `json:"req_extra"`
 	Context          context.Context
 	lastTouched      time.Time // record the start time of process
 	updateBranchSync bool
@@ -100,7 +101,7 @@ func TransFromDtmRequest(ctx context.Context, c *dtmgpb.DtmRequest) *TransGlobal
 		Protocol:       "grpc",
 		BinPayloads:    c.BinPayloads,
 		CustomData:     c.CustomedData,
-		RollbackReason: o.RollbackReason,
+		RollbackReason: c.RollbackReason,
 		TransOptions: dtmcli.TransOptions{
 			WaitResult:         o.WaitResult,
 			TimeoutToFail:      o.TimeoutToFail,
@@ -108,9 +109,9 @@ func TransFromDtmRequest(ctx context.Context, c *dtmgpb.DtmRequest) *TransGlobal
 			PassthroughHeaders: o.PassthroughHeaders,
 			BranchHeaders:      o.BranchHeaders,
 			RequestTimeout:     o.RequestTimeout,
-			RollbackReason:     o.RollbackReason,
 		},
 	}}
+	r.ReqExtra = c.ReqExtra
 	if c.Steps != "" {
 		dtmimp.MustUnmarshalString(c.Steps, &r.Steps)
 	}

dtmsvr/trans_process.go

@@ -7,6 +7,7 @@
 package dtmsvr
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -34,7 +35,7 @@ func (t *TransGlobal) process(branches []TransBranch) error {
 	if !t.WaitResult {
 		go func() {
 			err := t.processInner(branches)
-			if err != nil {
+			if err != nil && !errors.Is(err, dtmimp.ErrOngoing) {
 				logger.Errorf("processInner err: %v", err)
 			}
 		}()
@@ -58,7 +59,7 @@ func (t *TransGlobal) process(branches []TransBranch) error {
 func (t *TransGlobal) processInner(branches []TransBranch) (rerr error) {
 	defer handlePanic(&rerr)
 	defer func() {
-		if rerr != nil && rerr != dtmcli.ErrOngoing {
+		if rerr != nil && !errors.Is(rerr, dtmcli.ErrOngoing) {
 			logger.Errorf("processInner got error: %s", rerr.Error())
 		}
 		if TransProcessedTestChan != nil {

dtmsvr/trans_type_workflow.go

@@ -0,0 +1,43 @@
+package dtmsvr
+
+import (
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow/wfpb"
+)
+
+type transWorkflowProcessor struct {
+	*TransGlobal
+}
+
+func init() {
+	registorProcessorCreator("workflow", func(trans *TransGlobal) transProcessor { return &transWorkflowProcessor{TransGlobal: trans} })
+}
+
+func (t *transWorkflowProcessor) GenBranches() []TransBranch {
+	return []TransBranch{}
+}
+
+type cWorkflowCustom struct {
+	Name string `json:"name"`
+	Data []byte `json:"data"`
+}
+
+func (t *transWorkflowProcessor) ProcessOnce(branches []TransBranch) error {
+	if t.Status == dtmcli.StatusSubmitted { // client workflow finished
+		t.changeStatus(dtmcli.StatusSucceed)
+		return nil
+	} else if t.Status == dtmcli.StatusFailed || t.Status == dtmcli.StatusSucceed {
+		return nil
+	}
+
+	cmc := cWorkflowCustom{}
+	dtmimp.MustUnmarshalString(t.CustomData, &cmc)
+	data := cmc.Data
+	if t.Protocol == dtmimp.ProtocolGRPC {
+		wd := wfpb.WorkflowData{Data: cmc.Data}
+		data = dtmgimp.MustProtoMarshal(&wd)
+	}
+	return t.getURLResult(t.QueryPrepared, "00", cmc.Name, data)
+}

helper/test-cover.sh

@@ -1,13 +1,15 @@
 set -x
-echo "" > coverage.txt
+echo "mode: count" > coverage.txt
-for store in redis mysql boltdb postgres; do
+for store in redis boltdb mysql postgres; do
   for d in $(go list ./... | grep -v vendor); do
-    TEST_STORE=$store go test -covermode count -coverprofile=profile.out -coverpkg=github.com/dtm-labs/dtm/dtmcli,github.com/dtm-labs/dtm/dtmcli/dtmimp,github.com/dtm-labs/dtm/dtmcli/logger,github.com/dtm-labs/dtm/dtmgrpc,github.com/dtm-labs/dtm/dtmgrpc/dtmgimp,github.com/dtm-labs/dtm/dtmsvr,github.com/dtm-labs/dtm/dtmsvr/config,github.com/dtm-labs/dtm/dtmsvr/storage,github.com/dtm-labs/dtm/dtmsvr/storage/boltdb,github.com/dtm-labs/dtm/dtmsvr/storage/redis,github.com/dtm-labs/dtm/dtmsvr/storage/registry,github.com/dtm-labs/dtm/dtmsvr/storage/sql,github.com/dtm-labs/dtm/dtmutil -gcflags=-l $d || exit 1
+    TEST_STORE=$store go test -failfast -covermode count -coverprofile=profile.out -coverpkg=github.com/dtm-labs/dtm/dtmcli,github.com/dtm-labs/dtm/dtmcli/dtmimp,github.com/dtm-labs/dtm/dtmcli/logger,github.com/dtm-labs/dtm/dtmgrpc,github.com/dtm-labs/dtm/dtmgrpc/workflow,github.com/dtm-labs/dtm/dtmgrpc/dtmgimp,github.com/dtm-labs/dtm/dtmsvr,github.com/dtm-labs/dtm/dtmsvr/config,github.com/dtm-labs/dtm/dtmsvr/storage,github.com/dtm-labs/dtm/dtmsvr/storage/boltdb,github.com/dtm-labs/dtm/dtmsvr/storage/redis,github.com/dtm-labs/dtm/dtmsvr/storage/registry,github.com/dtm-labs/dtm/dtmsvr/storage/sql,github.com/dtm-labs/dtm/dtmutil -gcflags=-l $d || exit 1
       if [ -f profile.out ]; then
-          cat profile.out >> coverage.txt
+          cat profile.out | grep -v 'mode:' >> coverage.txt
           echo > profile.out
       fi
   done
 done
 
+# go tool cover -html=coverage.txt
+
 curl -s https://codecov.io/bash | bash

sqls/dtmsvr.storage.mysql.sql

@@ -7,14 +7,14 @@ CREATE TABLE if not EXISTS dtm.trans_global (
   `gid` varchar(128) NOT NULL COMMENT 'global transaction id',
   `trans_type` varchar(45) not null COMMENT 'transaction type: saga | xa | tcc | msg',
   `status` varchar(12) NOT NULL COMMENT 'tranaction status: prepared | submitted | aborting | finished | rollbacked',
-  `query_prepared` varchar(1024) NOT NULL COMMENT 'url to check for 2-phase message',
+  `query_prepared` varchar(1024) NOT NULL COMMENT 'url to check for msg|workflow',
   `protocol` varchar(45) not null comment 'protocol: http | grpc | json-rpc',
   `create_time` datetime DEFAULT NULL,
   `update_time` datetime DEFAULT NULL,
   `finish_time` datetime DEFAULT NULL,
   `rollback_time` datetime DEFAULT NULL,
   `options` varchar(1024) DEFAULT 'options for transaction like: TimeoutToFail, RequestTimeout',
-  `custom_data` varchar(256) DEFAULT '' COMMENT 'custom data for transaction',
+  `custom_data` varchar(1024) DEFAULT '' COMMENT 'custom data for transaction',
   `next_cron_interval` int(11) default null comment 'next cron interval. for use of cron job',
   `next_cron_time` datetime default null comment 'next time to process this trans. for use of cron job',
   `owner` varchar(128) not null default '' comment 'who is locking this trans',

sqls/dtmsvr.storage.postgres.sql

@@ -13,7 +13,7 @@ CREATE TABLE if not EXISTS trans_global (
   finish_time timestamp(0) with time zone DEFAULT NULL,
   rollback_time timestamp(0) with time zone DEFAULT NULL,
   options varchar(1024) DEFAULT '',
-  custom_data varchar(256) DEFAULT '',
+  custom_data varchar(1024) DEFAULT '',
   next_cron_interval int default null,
   next_cron_time timestamp(0) with time zone default null,
   owner varchar(128) not null default '',

sqls/dtmsvr.storage.tdsql.sql

@@ -14,7 +14,7 @@ CREATE TABLE if not EXISTS dtm.trans_global (
   `finish_time` datetime DEFAULT NULL,
   `rollback_time` datetime DEFAULT NULL,
   `options` varchar(1024) DEFAULT 'options for transaction like: TimeoutToFail, RequestTimeout',
-  `custom_data` varchar(256) DEFAULT '' COMMENT 'custom data for transaction',
+  `custom_data` varchar(1024) DEFAULT '' COMMENT 'custom data for transaction',
   `next_cron_interval` int(11) default null comment 'next cron interval. for use of cron job',
   `next_cron_time` datetime default null comment 'next time to process this trans. for use of cron job',
   `owner` varchar(128) not null default '' comment 'who is locking this trans',

test/busi/base_grpc.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
 	"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow"
 	grpc "google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	emptypb "google.golang.org/protobuf/types/known/emptypb"
@@ -32,22 +33,32 @@ var BusiGrpc = fmt.Sprintf("localhost:%d", BusiGrpcPort)
 // DtmClient grpc client for dtm
 var DtmClient dtmgpb.DtmClient
 
+// BusiCli grpc client for busi
+var BusiCli BusiClient
+
 // GrpcStartup for grpc
-func GrpcStartup() {
+func GrpcStartup() *grpc.Server {
 	conn, err := grpc.Dial(dtmutil.DefaultGrpcServer, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithUnaryInterceptor(dtmgimp.GrpcClientLog))
 	logger.FatalIfError(err)
 	DtmClient = dtmgpb.NewDtmClient(conn)
 	logger.Debugf("dtm client inited")
 
-	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", BusiGrpcPort))
+	conn1, err := grpc.Dial(BusiGrpc, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithUnaryInterceptor(workflow.Interceptor))
 	logger.FatalIfError(err)
+	BusiCli = NewBusiClient(conn1)
+
 	s := grpc.NewServer(grpc.UnaryInterceptor(dtmgimp.GrpcServerLog))
 	RegisterBusiServer(s, &busiServer{})
-	go func() {
+	return s
-		logger.Debugf("busi grpc listening at %v", lis.Addr())
+}
-		err := s.Serve(lis)
+
-		logger.FatalIfError(err)
+// GrpcServe start to serve grpc
-	}()
+func GrpcServe(server *grpc.Server) {
+	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", BusiGrpcPort))
+	logger.FatalIfError(err)
+	logger.Debugf("busi grpc listening at %v", lis.Addr())
+	err = server.Serve(lis)
+	logger.FatalIfError(err)
 }
 
 // busiServer is used to implement busi.BusiServer.

test/busi/base_http.go

@@ -10,10 +10,12 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"io/ioutil"
 
 	"github.com/dtm-labs/dtm/dtmcli"
 	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
 	"github.com/dtm-labs/dtm/dtmcli/logger"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow"
 	"github.com/dtm-labs/dtm/dtmutil"
 	"github.com/gin-gonic/gin"
 	"gorm.io/driver/mysql"
@@ -77,6 +79,11 @@ func BaseAppStartup() *gin.Engine {
 
 // BaseAddRoute add base route handler
 func BaseAddRoute(app *gin.Engine) {
+	app.POST(BusiAPI+"/workflow/resume", dtmutil.WrapHandler(func(ctx *gin.Context) interface{} {
+		data, err := ioutil.ReadAll(ctx.Request.Body)
+		logger.FatalIfError(err)
+		return workflow.ExecuteByQS(ctx.Request.URL.Query(), data)
+	}))
 	app.POST(BusiAPI+"/TransIn", dtmutil.WrapHandler(func(c *gin.Context) interface{} {
 		return handleGeneralBusiness(c, MainSwitch.TransInResult.Fetch(), reqFrom(c).TransInResult, "transIn")
 	}))
@@ -151,7 +158,7 @@ func BaseAddRoute(app *gin.Engine) {
 		tcc, err := dtmcli.TccFromQuery(c.Request.URL.Query())
 		logger.FatalIfError(err)
 		logger.Debugf("TransInTccNested ")
-		resp, err := tcc.CallBranch(&TransReq{Amount: reqFrom(c).Amount}, Busi+"/TransIn", Busi+"/TransInConfirm", Busi+"/TransInRevert")
+		resp, err := tcc.CallBranch(&ReqHTTP{Amount: reqFrom(c).Amount}, Busi+"/TransIn", Busi+"/TransInConfirm", Busi+"/TransInRevert")
 		if err != nil {
 			return err
 		}

test/busi/base_types.go

@@ -60,29 +60,29 @@ func GetBalanceByUID(uid int, store string) int {
 	return dtmimp.MustAtoi(ua.Balance[:len(ua.Balance)-3])
 }
 
-// TransReq transaction request payload
+// ReqHTTP transaction request payload
-type TransReq struct {
+type ReqHTTP struct {
 	Amount         int    `json:"amount"`
 	TransInResult  string `json:"trans_in_result"`
 	TransOutResult string `json:"trans_out_Result"`
 	Store          string `json:"store"` // default mysql, value can be mysql|redis
 }
 
-func (t *TransReq) String() string {
+func (t *ReqHTTP) String() string {
 	return fmt.Sprintf("amount: %d transIn: %s transOut: %s", t.Amount, t.TransInResult, t.TransOutResult)
 }
 
-// GenTransReq 1
+// GenReqHTTP 1
-func GenTransReq(amount int, outFailed bool, inFailed bool) *TransReq {
+func GenReqHTTP(amount int, outFailed bool, inFailed bool) *ReqHTTP {
-	return &TransReq{
+	return &ReqHTTP{
 		Amount:         amount,
 		TransOutResult: dtmimp.If(outFailed, dtmcli.ResultFailure, "").(string),
 		TransInResult:  dtmimp.If(inFailed, dtmcli.ResultFailure, "").(string),
 	}
 }
 
-// GenBusiReq 1
+// GenReqGrpc 1
-func GenBusiReq(amount int, outFailed bool, inFailed bool) *BusiReq {
+func GenReqGrpc(amount int, outFailed bool, inFailed bool) *ReqGrpc {
 	return &BusiReq{
 		Amount:         int64(amount),
 		TransOutResult: dtmimp.If(outFailed, dtmcli.ResultFailure, "").(string),
@@ -90,16 +90,16 @@ func GenBusiReq(amount int, outFailed bool, inFailed bool) *BusiReq {
 	}
 }
 
-func reqFrom(c *gin.Context) *TransReq {
+func reqFrom(c *gin.Context) *ReqHTTP {
 	v, ok := c.Get("trans_req")
 	if !ok {
-		req := TransReq{}
+		req := ReqHTTP{}
 		err := c.BindJSON(&req)
 		logger.FatalIfError(err)
 		c.Set("trans_req", &req)
 		v = &req
 	}
-	return v.(*TransReq)
+	return v.(*ReqHTTP)
 }
 
 func infoFromContext(c *gin.Context) *dtmcli.BranchBarrier {

test/busi/base_workflow.go

@@ -0,0 +1,13 @@
+package busi
+
+import (
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow"
+	"github.com/dtm-labs/dtm/dtmutil"
+	"google.golang.org/grpc"
+)
+
+// WorkflowStarup 1
+func WorkflowStarup(server *grpc.Server) {
+	workflow.InitHTTP(dtmServer, Busi+"/workflow/resume")
+	workflow.InitGrpc(dtmutil.DefaultGrpcServer, BusiGrpc, server)
+}

test/busi/busi.go → test/busi/data.go

@@ -9,6 +9,7 @@ import (
 	"github.com/dtm-labs/dtm/dtmcli"
 	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
 	"github.com/dtm-labs/dtm/dtmcli/logger"
+	"github.com/dtm-labs/dtm/dtmutil"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -16,6 +17,21 @@ import (
 	status "google.golang.org/grpc/status"
 )
 
+// PopulateDB populate example mysql data
+func PopulateDB(skipDrop bool) {
+	resetXaData()
+	file := fmt.Sprintf("%s/busi.%s.sql", dtmutil.GetSQLDir(), BusiConf.Driver)
+	dtmutil.RunSQLScript(BusiConf, file, skipDrop)
+	file = fmt.Sprintf("%s/dtmcli.barrier.%s.sql", dtmutil.GetSQLDir(), BusiConf.Driver)
+	dtmutil.RunSQLScript(BusiConf, file, skipDrop)
+	file = fmt.Sprintf("%s/dtmsvr.storage.%s.sql", dtmutil.GetSQLDir(), BusiConf.Driver)
+	dtmutil.RunSQLScript(BusiConf, file, skipDrop)
+	_, err := RedisGet().FlushAll(context.Background()).Result() // redis barrier need clear
+	dtmimp.E2P(err)
+	SetRedisBothAccount(10000, 10000)
+	SetupMongoBarrierAndBusi()
+}
+
 // TransOutUID 1
 const TransOutUID = 1
 

test/busi/startup.go

@@ -1,31 +1,14 @@
 package busi
 
 import (
-	"context"
-	"fmt"
-
-	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
-	"github.com/dtm-labs/dtm/dtmutil"
 	"github.com/gin-gonic/gin"
 )
 
 // Startup startup the busi's grpc and http service
 func Startup() *gin.Engine {
-	GrpcStartup()
+	svr := GrpcStartup()
-	return BaseAppStartup()
+	app := BaseAppStartup()
-}
+	WorkflowStarup(svr)
-
+	go GrpcServe(svr)
-// PopulateDB populate example mysql data
+	return app
-func PopulateDB(skipDrop bool) {
-	resetXaData()
-	file := fmt.Sprintf("%s/busi.%s.sql", dtmutil.GetSQLDir(), BusiConf.Driver)
-	dtmutil.RunSQLScript(BusiConf, file, skipDrop)
-	file = fmt.Sprintf("%s/dtmcli.barrier.%s.sql", dtmutil.GetSQLDir(), BusiConf.Driver)
-	dtmutil.RunSQLScript(BusiConf, file, skipDrop)
-	file = fmt.Sprintf("%s/dtmsvr.storage.%s.sql", dtmutil.GetSQLDir(), BusiConf.Driver)
-	dtmutil.RunSQLScript(BusiConf, file, skipDrop)
-	_, err := RedisGet().FlushAll(context.Background()).Result() // redis barrier need clear
-	dtmimp.E2P(err)
-	SetRedisBothAccount(10000, 10000)
-	SetupMongoBarrierAndBusi()
 }

test/busi/utils.go

@@ -25,6 +25,9 @@ import (
 	"google.golang.org/grpc/metadata"
 )
 
+// ReqGrpc is the req for grpc protocol
+type ReqGrpc = BusiReq
+
 func dbGet() *dtmutil.DB {
 	return dtmutil.DbGet(BusiConf)
 }
@@ -84,7 +87,7 @@ func SetGrpcHeaderForHeadersYes(ctx context.Context, method string, req, reply i
 
 // SetHTTPHeaderForHeadersYes interceptor to set head for HeadersYes
 func SetHTTPHeaderForHeadersYes(c *resty.Client, r *resty.Request) error {
-	if b, ok := r.Body.(*dtmcli.Saga); ok && strings.HasSuffix(b.Gid, "HeadersYes") {
+	if b, ok := r.Body.(*dtmimp.TransBase); ok && strings.HasSuffix(b.Gid, "HeadersYes") {
 		logger.Debugf("set test_header for url: %s", r.URL)
 		r.SetHeader("test_header", "yes")
 	}

test/main_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
 	"github.com/dtm-labs/dtm/dtmcli/logger"
 	"github.com/dtm-labs/dtm/dtmgrpc"
 	"github.com/dtm-labs/dtm/dtmsvr"
@@ -39,7 +40,7 @@ func TestMain(m *testing.M) {
 	dtmcli.GetRestyClient().OnBeforeRequest(busi.SetHTTPHeaderForHeadersYes)
 	dtmcli.GetRestyClient().OnAfterResponse(func(c *resty.Client, resp *resty.Response) error { return nil })
 
-	tenv := os.Getenv("TEST_STORE")
+	tenv := dtmimp.OrString(os.Getenv("TEST_STORE"), config.Redis)
 	conf.Store.Host = "localhost"
 	conf.Store.Driver = tenv
 	if tenv == "boltdb" {

test/msg_barrier_mongo_test.go

@@ -14,7 +14,7 @@ import (
 func TestMsgMongoDoSucceed(t *testing.T) {
 	before := getBeforeBalances("mongo")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaMongoTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/MongoQueryPrepared", func(bb *dtmcli.BranchBarrier) error {
@@ -32,7 +32,7 @@ func TestMsgMongoDoSucceed(t *testing.T) {
 func TestMsgMongoDoBusiFailed(t *testing.T) {
 	before := getBeforeBalances("mongo")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaMongoTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/MongoQueryPrepared", func(bb *dtmcli.BranchBarrier) error {
@@ -45,7 +45,7 @@ func TestMsgMongoDoBusiFailed(t *testing.T) {
 func TestMsgMongoDoBusiLater(t *testing.T) {
 	before := getBeforeBalances("mongo")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	_, err := dtmcli.GetRestyClient().R().
 		SetQueryParams(map[string]string{
 			"trans_type": "msg",
@@ -70,7 +70,7 @@ func TestMsgMongoDoBusiLater(t *testing.T) {
 func TestMsgMongoDoCommitFailed(t *testing.T) {
 	before := getBeforeBalances("mongo")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaMongoTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/MongoQueryPrepared", func(bb *dtmcli.BranchBarrier) error {
@@ -87,7 +87,7 @@ func TestMsgMongoDoCommitFailed(t *testing.T) {
 func TestMsgMongoDoCommitAfterFailed(t *testing.T) {
 	before := getBeforeBalances("mongo")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaMongoTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/MongoQueryPrepared", func(bb *dtmcli.BranchBarrier) error {

test/msg_barrier_redis_test.go

@@ -13,7 +13,7 @@ import (
 func TestMsgRedisDo(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaRedisTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/RedisQueryPrepared", func(bb *dtmcli.BranchBarrier) error {
@@ -29,7 +29,7 @@ func TestMsgRedisDo(t *testing.T) {
 func TestMsgRedisDoBusiFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaRedisTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/RedisQueryPrepared", func(bb *dtmcli.BranchBarrier) error {
@@ -42,7 +42,7 @@ func TestMsgRedisDoBusiFailed(t *testing.T) {
 func TestMsgRedisDoBusiLater(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	_, err := dtmcli.GetRestyClient().R().
 		SetQueryParams(map[string]string{
 			"trans_type": "msg",
@@ -65,7 +65,7 @@ func TestMsgRedisDoBusiLater(t *testing.T) {
 func TestMsgRedisDoPrepareFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer+"not-exists", gid).
 		Add(busi.Busi+"/SagaRedisTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/RedisQueryPrepared", func(bb *dtmcli.BranchBarrier) error {
@@ -78,7 +78,7 @@ func TestMsgRedisDoPrepareFailed(t *testing.T) {
 func TestMsgRedisDoCommitFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaRedisTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/RedisQueryPrepared", func(bb *dtmcli.BranchBarrier) error {
@@ -91,7 +91,7 @@ func TestMsgRedisDoCommitFailed(t *testing.T) {
 func TestMsgRedisDoCommitAfterFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaRedisTransIn", req)
 	err := msg.DoAndSubmit(Busi+"/RedisQueryPrepared", func(bb *dtmcli.BranchBarrier) error {

test/msg_barrier_test.go

@@ -17,7 +17,7 @@ import (
 func TestMsgDoAndSubmit(t *testing.T) {
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	err := msg.DoAndSubmitDB(Busi+"/QueryPreparedB", dbGet().ToSQLDB(), func(tx *sql.Tx) error {
@@ -33,7 +33,7 @@ func TestMsgDoAndSubmit(t *testing.T) {
 func TestMsgDoAndSubmitBusiFailed(t *testing.T) {
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	err := msg.DoAndSubmitDB(Busi+"/QueryPreparedB", dbGet().ToSQLDB(), func(tx *sql.Tx) error {
@@ -46,7 +46,7 @@ func TestMsgDoAndSubmitBusiFailed(t *testing.T) {
 func TestMsgDoAndSubmitBusiLater(t *testing.T) {
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	_, err := dtmcli.GetRestyClient().R().
 		SetQueryParams(map[string]string{
 			"trans_type": "msg",
@@ -69,7 +69,7 @@ func TestMsgDoAndSubmitBusiLater(t *testing.T) {
 func TestMsgDoAndSubmitPrepareFailed(t *testing.T) {
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer+"not-exists", gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	err := msg.DoAndSubmitDB(Busi+"/QueryPreparedB", dbGet().ToSQLDB(), func(tx *sql.Tx) error {
@@ -85,7 +85,7 @@ func TestMsgDoAndSubmitCommitFailed(t *testing.T) {
 	}
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	var g *monkey.PatchGuard
@@ -108,7 +108,7 @@ func TestMsgDoAndSubmitCommitAfterFailed(t *testing.T) {
 	}
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(DtmServer, gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	var guard *monkey.PatchGuard

test/msg_delay_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func genMsgDelay(gid string) *dtmcli.Msg {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(dtmutil.DefaultHTTPServer, gid).
 		Add(busi.Busi+"/TransOut", &req).
 		Add(busi.Busi+"/TransIn", &req).SetDelay(10)

test/msg_grpc_barrier_redis_test.go

@@ -14,7 +14,7 @@ import (
 func TestMsgGrpcRedisDo(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
 		Add(busi.BusiGrpc+"/busi.Busi/TransInRedis", req)
 	err := msg.DoAndSubmit(busi.BusiGrpc+"/busi.Busi/QueryPreparedRedis", func(bb *dtmcli.BranchBarrier) error {
@@ -30,7 +30,7 @@ func TestMsgGrpcRedisDo(t *testing.T) {
 func TestMsgGrpcRedisDoBusiFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
 		Add(busi.BusiGrpc+"/busi.Busi/TransInRedis", req)
 	err := msg.DoAndSubmit(busi.BusiGrpc+"/busi.Busi/QueryPreparedRedis", func(bb *dtmcli.BranchBarrier) error {
@@ -43,7 +43,7 @@ func TestMsgGrpcRedisDoBusiFailed(t *testing.T) {
 func TestMsgGrpcRedisDoPrepareFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer+"not-exists", gid).
 		Add(busi.BusiGrpc+"/busi.Busi/TransInRedis", req)
 	err := msg.DoAndSubmit(busi.BusiGrpc+"/busi.Busi/QueryPreparedRedis", func(bb *dtmcli.BranchBarrier) error {
@@ -56,7 +56,7 @@ func TestMsgGrpcRedisDoPrepareFailed(t *testing.T) {
 func TestMsgGrpcRedisDoCommitFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
 		Add(busi.BusiGrpc+"/busi.Busi/TransInRedis", req)
 	err := msg.DoAndSubmit(busi.BusiGrpc+"/busi.Busi/QueryPreparedRedis", func(bb *dtmcli.BranchBarrier) error {
@@ -69,7 +69,7 @@ func TestMsgGrpcRedisDoCommitFailed(t *testing.T) {
 func TestMsgGrpcRedisDoCommitAfterFailed(t *testing.T) {
 	before := getBeforeBalances("redis")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
 		Add(busi.BusiGrpc+"/busi.Busi/TransInRedis", req)
 	err := msg.DoAndSubmit(busi.BusiGrpc+"/busi.Busi/QueryPreparedRedis", func(bb *dtmcli.BranchBarrier) error {

test/msg_grpc_barrier_test.go

@@ -17,7 +17,7 @@ import (
 func TestMsgGrpcPrepareAndSubmit(t *testing.T) {
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
 		Add(busi.BusiGrpc+"/busi.Busi/TransInBSaga", req)
 	err := msg.DoAndSubmitDB(busi.BusiGrpc+"/busi.Busi/QueryPreparedB", dbGet().ToSQLDB(), func(tx *sql.Tx) error {
@@ -36,7 +36,7 @@ func TestMsgGrpcPrepareAndSubmitCommitAfterFailed(t *testing.T) {
 	}
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
 		Add(busi.BusiGrpc+"/busi.Busi/TransInBSaga", req)
 	var guard *monkey.PatchGuard
@@ -60,7 +60,7 @@ func TestMsgGrpcPrepareAndSubmitCommitFailed(t *testing.T) {
 	}
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	var g *monkey.PatchGuard

test/msg_jrpc_test.go

@@ -49,7 +49,7 @@ func TestMsgJrpcResults(t *testing.T) {
 func TestMsgJrpcDoAndSubmit(t *testing.T) {
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(dtmutil.DefaultJrpcServer, gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	msg.Protocol = dtmimp.Jrpc
@@ -66,7 +66,7 @@ func TestMsgJrpcDoAndSubmit(t *testing.T) {
 func TestMsgJrpcDoAndSubmitBusiFailed(t *testing.T) {
 	before := getBeforeBalances("mysql")
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(dtmutil.DefaultJrpcServer, gid).
 		Add(busi.Busi+"/SagaBTransIn", req)
 	msg.Protocol = dtmimp.Jrpc
@@ -130,12 +130,12 @@ func TestMsgJprcAbnormal(t *testing.T) {
 func TestMsgJprcAbnormal2(t *testing.T) {
 	tb := dtmimp.NewTransBase(dtmimp.GetFuncName(), "msg", dtmutil.DefaultJrpcServer, "01")
 	tb.Protocol = "json-rpc"
-	err := dtmimp.TransCallDtm(tb, "", "newGid")
+	_, err := dtmimp.TransCallDtmExt(tb, "", "newGid")
 	assert.Nil(t, err)
 }
 
 func genJrpcMsg(gid string) *dtmcli.Msg {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(dtmutil.DefaultJrpcServer, gid).
 		Add(busi.Busi+"/TransOut", &req).
 		Add(busi.BusiJrpcURL+"TransIn", &req)

test/msg_test.go

@@ -68,7 +68,7 @@ func TestMsgAbnormal(t *testing.T) {
 }
 
 func genMsg(gid string) *dtmcli.Msg {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	msg := dtmcli.NewMsg(dtmutil.DefaultHTTPServer, gid).
 		Add(busi.Busi+"/TransOut", &req).
 		Add(busi.Busi+"/TransIn", &req)

test/saga_barrier_mongo_test.go

@@ -38,7 +38,7 @@ func TestSagaBarrierMongoRollback(t *testing.T) {
 }
 
 func genSagaBarrierMongo(gid string, transInFailed bool) *dtmcli.Saga {
-	req := busi.GenTransReq(30, false, transInFailed)
+	req := busi.GenReqHTTP(30, false, transInFailed)
 	req.Store = "mongo"
 	return dtmcli.NewSaga(DtmServer, gid).
 		Add(Busi+"/SagaMongoTransOut", Busi+"/SagaMongoTransOutCom", req).

test/saga_barrier_redis_test.go

@@ -40,7 +40,7 @@ func TestSagaBarrierRedisRollback(t *testing.T) {
 }
 
 func genSagaBarrierRedis(gid string) *dtmcli.Saga {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	req.Store = "redis"
 	return dtmcli.NewSaga(DtmServer, gid).
 		Add(Busi+"/SagaRedisTransIn", Busi+"/SagaRedisTransInCom", req).

test/saga_barrier_test.go

@@ -34,14 +34,14 @@ func TestSagaBarrierRollback(t *testing.T) {
 }
 
 func genSagaBarrier(gid string, outFailed, inFailed bool) *dtmcli.Saga {
-	req := busi.GenTransReq(30, outFailed, inFailed)
+	req := busi.GenReqHTTP(30, outFailed, inFailed)
 	return dtmcli.NewSaga(DtmServer, gid).
 		Add(Busi+"/SagaBTransOut", Busi+"/SagaBTransOutCom", req).
 		Add(Busi+"/SagaBTransIn", Busi+"/SagaBTransInCom", req)
 }
 
 func TestSagaBarrier2Normal(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	saga := dtmcli.NewSaga(DtmServer, gid).
 		Add(Busi+"/SagaBTransOut", Busi+"/SagaBTransOutCom", req).

test/saga_grpc_barrier_test.go

@@ -36,7 +36,7 @@ func TestSagaGrpcBarrierRollback(t *testing.T) {
 
 func genSagaGrpcBarrier(gid string, outFailed bool, inFailed bool) *dtmgrpc.SagaGrpc {
 	saga := dtmgrpc.NewSagaGrpc(dtmutil.DefaultGrpcServer, gid)
-	req := busi.GenBusiReq(30, outFailed, inFailed)
+	req := busi.GenReqGrpc(30, outFailed, inFailed)
 	saga.Add(busi.BusiGrpc+"/busi.Busi/TransOutBSaga", busi.BusiGrpc+"/busi.Busi/TransOutRevertBSaga", req)
 	saga.Add(busi.BusiGrpc+"/busi.Busi/TransInBSaga", busi.BusiGrpc+"/busi.Busi/TransInRevertBSaga", req)
 	return saga

test/saga_grpc_test.go

@@ -83,7 +83,7 @@ func TestSagaGrpcNormalWait(t *testing.T) {
 
 func TestSagaGrpcEmptyUrl(t *testing.T) {
 	saga := dtmgrpc.NewSagaGrpc(dtmutil.DefaultGrpcServer, dtmimp.GetFuncName())
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	saga.Add(busi.BusiGrpc+"/busi.Busi/TransOut", busi.BusiGrpc+"/busi.Busi/TransOutRevert", req)
 	saga.Add("", busi.BusiGrpc+"/busi.Busi/TransInRevert", req)
 	saga.Submit()
@@ -95,7 +95,7 @@ func TestSagaGrpcEmptyUrl(t *testing.T) {
 //nolint: unparam
 func genSagaGrpc(gid string, outFailed bool, inFailed bool) *dtmgrpc.SagaGrpc {
 	saga := dtmgrpc.NewSagaGrpc(dtmutil.DefaultGrpcServer, gid)
-	req := busi.GenBusiReq(30, outFailed, inFailed)
+	req := busi.GenReqGrpc(30, outFailed, inFailed)
 	saga.Add(busi.BusiGrpc+"/busi.Busi/TransOut", busi.BusiGrpc+"/busi.Busi/TransOutRevert", req)
 	saga.Add(busi.BusiGrpc+"/busi.Busi/TransIn", busi.BusiGrpc+"/busi.Busi/TransInRevert", req)
 	return saga

test/saga_test.go

@@ -78,7 +78,7 @@ func TestSagaAbnormal(t *testing.T) {
 
 func TestSagaEmptyUrl(t *testing.T) {
 	saga := dtmcli.NewSaga(dtmutil.DefaultHTTPServer, dtmimp.GetFuncName())
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	saga.Add(busi.Busi+"/TransOut", "", &req)
 	saga.Add("", "", &req)
 	saga.Submit()
@@ -89,7 +89,7 @@ func TestSagaEmptyUrl(t *testing.T) {
 
 func genSaga(gid string, outFailed bool, inFailed bool) *dtmcli.Saga {
 	saga := dtmcli.NewSaga(dtmutil.DefaultHTTPServer, gid)
-	req := busi.GenTransReq(30, outFailed, inFailed)
+	req := busi.GenReqHTTP(30, outFailed, inFailed)
 	saga.Add(busi.Busi+"/TransOut", busi.Busi+"/TransOutRevert", &req)
 	saga.Add(busi.Busi+"/TransIn", busi.Busi+"/TransInRevert", &req)
 	return saga
@@ -97,7 +97,7 @@ func genSaga(gid string, outFailed bool, inFailed bool) *dtmcli.Saga {
 
 func genSaga1(gid string, outFailed bool, inFailed bool) *dtmcli.Saga {
 	saga := dtmcli.NewSaga(dtmutil.DefaultHTTPServer, gid)
-	req := busi.GenTransReq(30, outFailed, inFailed)
+	req := busi.GenReqHTTP(30, outFailed, inFailed)
 	saga.Add(busi.Busi+"/TransOut", busi.Busi+"/TransOutRevert", &req)
 	return saga
 }

test/tcc_barrier_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 func TestTccBarrierNormal(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction(DtmServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, err := tcc.CallBranch(req, Busi+"/TccBTransOutTry", Busi+"/TccBTransOutConfirm", Busi+"/TccBTransOutCancel")
@@ -36,7 +36,7 @@ func TestTccBarrierNormal(t *testing.T) {
 }
 
 func TestTccBarrierRollback(t *testing.T) {
-	req := busi.GenTransReq(30, false, true)
+	req := busi.GenReqHTTP(30, false, true)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction(DtmServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, err := tcc.CallBranch(req, Busi+"/TccBTransOutTry", Busi+"/TccBTransOutConfirm", Busi+"/TccBTransOutCancel")
@@ -69,7 +69,7 @@ func runTestTccBarrierDisorder(t *testing.T, store string) {
 	gid := dtmimp.GetFuncName() + store
 	cronFinished := make(chan string, 2)
 	err := dtmcli.TccGlobalTransaction(DtmServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
-		body := &busi.TransReq{Amount: 30, Store: store}
+		body := &busi.ReqHTTP{Amount: 30, Store: store}
 		tryURL := Busi + "/TccBTransOutTry"
 		confirmURL := Busi + "/TccBTransOutConfirm"
 		cancelURL := Busi + "/SleepCancel"

test/tcc_cover_test.go

@@ -32,7 +32,7 @@ func TestTccCoverPanic(t *testing.T) {
 }
 
 func TestTccNested(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction(dtmutil.DefaultHTTPServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, err := tcc.CallBranch(req, Busi+"/TransOut", Busi+"/TransOutConfirm", Busi+"/TransOutRevert")

test/tcc_grpc_cover_test.go

@@ -32,7 +32,7 @@ func TestTccGrpcCoverPanic(t *testing.T) {
 }
 
 func TestTccGrpcCoverCallBranch(t *testing.T) {
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmgrpc.TccGlobalTransaction(dtmutil.DefaultGrpcServer, gid, func(tcc *dtmgrpc.TccGrpc) error {
 

test/tcc_grpc_test.go

@@ -21,7 +21,7 @@ import (
 )
 
 func TestTccGrpcNormal(t *testing.T) {
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmgrpc.TccGlobalTransaction(dtmutil.DefaultGrpcServer, gid, func(tcc *dtmgrpc.TccGrpc) error {
 		r := &emptypb.Empty{}
@@ -38,7 +38,7 @@ func TestTccGrpcNormal(t *testing.T) {
 
 func TestTccGrpcRollback(t *testing.T) {
 	gid := dtmimp.GetFuncName()
-	req := busi.GenBusiReq(30, false, true)
+	req := busi.GenReqGrpc(30, false, true)
 	err := dtmgrpc.TccGlobalTransaction(dtmutil.DefaultGrpcServer, gid, func(tcc *dtmgrpc.TccGrpc) error {
 		r := &emptypb.Empty{}
 		err := tcc.CallBranch(req, busi.BusiGrpc+"/busi.Busi/TransOutTcc", busi.BusiGrpc+"/busi.Busi/TransOutConfirm", busi.BusiGrpc+"/busi.Busi/TransOutRevert", r)
@@ -56,7 +56,7 @@ func TestTccGrpcRollback(t *testing.T) {
 }
 
 func TestTccGrpcNested(t *testing.T) {
-	req := busi.GenBusiReq(30, false, false)
+	req := busi.GenReqGrpc(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmgrpc.TccGlobalTransaction(dtmutil.DefaultGrpcServer, gid, func(tcc *dtmgrpc.TccGrpc) error {
 		r := &emptypb.Empty{}

test/tcc_jrpc_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestTccJrpcNormal(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction2(dtmutil.DefaultJrpcServer, gid, func(tcc *dtmcli.Tcc) {
 		tcc.Protocol = dtmimp.Jrpc

test/tcc_old_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestTccOldNormal(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction(dtmutil.DefaultHTTPServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, err := tcc.CallBranch(req, Busi+"/TransOutOld", Busi+"/TransOutConfirmOld", Busi+"/TransOutRevertOld")
@@ -27,7 +27,7 @@ func TestTccOldNormal(t *testing.T) {
 
 func TestTccOldRollback(t *testing.T) {
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, true)
+	req := busi.GenReqHTTP(30, false, true)
 	err := dtmcli.TccGlobalTransaction(dtmutil.DefaultHTTPServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, rerr := tcc.CallBranch(req, Busi+"/TransOutOld", Busi+"/TransOutConfirmOld", Busi+"/TransOutRevertOld")
 		assert.Nil(t, rerr)
@@ -43,7 +43,7 @@ func TestTccOldRollback(t *testing.T) {
 }
 
 func TestTccOldTimeout(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	timeoutChan := make(chan int, 1)
 

test/tcc_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestTccNormal(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction(dtmutil.DefaultHTTPServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, err := tcc.CallBranch(req, Busi+"/TransOut", Busi+"/TransOutConfirm", Busi+"/TransOutRevert")
@@ -33,7 +33,7 @@ func TestTccNormal(t *testing.T) {
 
 func TestTccRollback(t *testing.T) {
 	gid := dtmimp.GetFuncName()
-	req := busi.GenTransReq(30, false, true)
+	req := busi.GenReqHTTP(30, false, true)
 	err := dtmcli.TccGlobalTransaction(dtmutil.DefaultHTTPServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, rerr := tcc.CallBranch(req, Busi+"/TransOut", Busi+"/TransOutConfirm", Busi+"/TransOutRevert")
 		assert.Nil(t, rerr)
@@ -50,7 +50,7 @@ func TestTccRollback(t *testing.T) {
 }
 
 func TestTccTimeout(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	timeoutChan := make(chan int, 1)
 
@@ -73,7 +73,7 @@ func TestTccTimeout(t *testing.T) {
 }
 
 func TestTccCompatible(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction(dtmutil.DefaultHTTPServer, gid, func(tcc *dtmcli.Tcc) (*resty.Response, error) {
 		_, err := tcc.CallBranch(req, Busi+"/TransOut", Busi+"/TransOutConfirm", Busi+"/TransOutRevert")
@@ -88,7 +88,7 @@ func TestTccCompatible(t *testing.T) {
 }
 
 func TestTccHeaders(t *testing.T) {
-	req := busi.GenTransReq(30, false, false)
+	req := busi.GenReqHTTP(30, false, false)
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.TccGlobalTransaction2(dtmutil.DefaultHTTPServer, gid, func(t *dtmcli.Tcc) {
 		t.BranchHeaders = map[string]string{

test/workflow_base_test.go

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021 yedf. All rights reserved.
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file.
+ */
+
+package test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmsvr"
+	"github.com/dtm-labs/dtm/dtmsvr/storage"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWorkflowBranchConflict(t *testing.T) {
+	gid := dtmimp.GetFuncName()
+	store := dtmsvr.GetStore()
+	now := time.Now()
+	g := &storage.TransGlobalStore{
+		Gid:          gid,
+		Status:       dtmcli.StatusPrepared,
+		NextCronTime: &now,
+	}
+	err := store.MaySaveNewTrans(g, []storage.TransBranchStore{
+		{
+			BranchID: "00",
+			Op:       dtmimp.OpAction,
+		},
+	})
+	assert.Nil(t, err)
+	err = dtmimp.CatchP(func() {
+		store.LockGlobalSaveBranches(gid, dtmcli.StatusPrepared, []storage.TransBranchStore{
+			{BranchID: "00", Op: dtmimp.OpAction},
+		}, -1)
+	})
+	assert.Error(t, err)
+	store.ChangeGlobalStatus(g, StatusSucceed, []string{}, true)
+}

test/workflow_grpc_test.go

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2021 yedf. All rights reserved.
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file.
+ */
+
+package test
+
+import (
+	"database/sql"
+	"testing"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow"
+	"github.com/dtm-labs/dtm/test/busi"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWorkflowGrpcSimple(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolHTTP)
+	req := &busi.ReqGrpc{Amount: 30, TransInResult: "FAILURE"}
+	gid := dtmimp.GetFuncName()
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.BusiReq
+		dtmgimp.MustProtoUnmarshal(data, &req)
+		_, err := busi.BusiCli.TransOutBSaga(wf.NewBranchCtx(), &req)
+		if err != nil {
+			return err
+		}
+		_, err = busi.BusiCli.TransInBSaga(wf.NewBranchCtx(), &req)
+		return err
+	})
+	err := workflow.Execute(gid, gid, dtmgimp.MustProtoMarshal(req))
+	assert.Error(t, err, dtmcli.ErrFailure)
+	assert.Equal(t, StatusFailed, getTransStatus(gid))
+	waitTransProcessed(gid)
+}
+
+func TestWorkflowGrpcNormal(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolGRPC)
+	req := &busi.ReqGrpc{Amount: 30, TransInResult: "FAILURE"}
+	gid := dtmimp.GetFuncName()
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.BusiReq
+		dtmgimp.MustProtoUnmarshal(data, &req)
+		wf.NewBranch().OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			_, err := busi.BusiCli.TransOutRevertBSaga(wf.Context, &req)
+			return err
+		})
+		_, err := busi.BusiCli.TransOutBSaga(wf.Context, &req)
+		if err != nil {
+			return err
+		}
+		wf.NewBranch().OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			_, err := busi.BusiCli.TransInRevertBSaga(wf.Context, &req)
+			return err
+		})
+		_, err = busi.BusiCli.TransInBSaga(wf.Context, &req)
+		return err
+	})
+	err := workflow.Execute(gid, gid, dtmgimp.MustProtoMarshal(req))
+	assert.Error(t, err, dtmcli.ErrFailure)
+	assert.Equal(t, StatusFailed, getTransStatus(gid))
+	waitTransProcessed(gid)
+}
+
+func TestWorkflowMixed(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolHTTP)
+	req := &busi.BusiReq{Amount: 30}
+	gid := dtmimp.GetFuncName()
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.BusiReq
+		dtmgimp.MustProtoUnmarshal(data, &req)
+
+		wf.NewBranch().OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			_, err := busi.BusiCli.TransOutRevertBSaga(wf.Context, &req)
+			return err
+		})
+		_, err := busi.BusiCli.TransOutBSaga(wf.Context, &req)
+		if err != nil {
+			return err
+		}
+
+		_, err = wf.NewBranch().OnCommit(func(bb *dtmcli.BranchBarrier) error {
+			_, err := busi.BusiCli.TransInConfirm(wf.Context, &req)
+			return err
+		}).OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			req2 := &busi.ReqHTTP{Amount: 30}
+			_, err := wf.NewRequest().SetBody(req2).Post(Busi + "/TransInRevert")
+			return err
+		}).Do(func(bb *dtmcli.BranchBarrier) ([]byte, error) {
+			err := busi.SagaAdjustBalance(dbGet().ToSQLDB(), busi.TransInUID, int(req.Amount), "")
+			return nil, err
+		})
+		if err != nil {
+			return err
+		}
+		_, err = wf.NewBranch().DoXa(busi.BusiConf, func(db *sql.DB) ([]byte, error) {
+			return nil, busi.SagaAdjustBalance(db, busi.TransInUID, 0, dtmcli.ResultSuccess)
+		})
+		return err
+	})
+	err := workflow.Execute(gid, gid, dtmgimp.MustProtoMarshal(req))
+	assert.Nil(t, err)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+	waitTransProcessed(gid)
+}
+
+func TestWorkflowGrpcError(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolGRPC)
+	req := &busi.BusiReq{Amount: 30}
+	gid := dtmimp.GetFuncName()
+	busi.MainSwitch.TransOutResult.SetOnce("ERROR")
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.BusiReq
+		dtmgimp.MustProtoUnmarshal(data, &req)
+		_, err := busi.BusiCli.TransOut(wf.NewBranchCtx(), &req)
+		if err != nil {
+			return err
+		}
+		_, err = busi.BusiCli.TransIn(wf.NewBranchCtx(), &req)
+		return err
+	})
+	err := workflow.Execute(gid, gid, dtmgimp.MustProtoMarshal(req))
+	assert.Error(t, err)
+	go waitTransProcessed(gid)
+	cronTransOnceForwardCron(t, gid, 1000)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+}

test/workflow_http_test.go

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2021 yedf. All rights reserved.
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file.
+ */
+
+package test
+
+import (
+	"database/sql"
+	"testing"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow"
+	"github.com/dtm-labs/dtm/test/busi"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWorkflowNormal(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolHTTP)
+	req := busi.GenReqHTTP(30, false, false)
+	gid := dtmimp.GetFuncName()
+
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.ReqHTTP
+		dtmimp.MustUnmarshal(data, &req)
+		_, err := wf.NewBranch().NewRequest().SetBody(req).Post(Busi + "/TransOut")
+		if err != nil {
+			return err
+		}
+		_, err = wf.NewBranch().NewRequest().SetBody(req).Post(Busi + "/TransIn")
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	err := workflow.Execute(gid, gid, dtmimp.MustMarshal(req))
+	assert.Nil(t, err)
+	waitTransProcessed(gid)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+}
+
+func TestWorkflowRollback(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolHTTP)
+
+	req := &busi.ReqHTTP{Amount: 30, TransInResult: dtmimp.ResultFailure}
+	gid := dtmimp.GetFuncName()
+
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.ReqHTTP
+		dtmimp.MustUnmarshal(data, &req)
+		_, err := wf.NewBranch().OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			_, err := wf.NewRequest().SetBody(req).Post(Busi + "/SagaBTransOutCom")
+			return err
+		}).Do(func(bb *dtmcli.BranchBarrier) ([]byte, error) {
+			return nil, bb.CallWithDB(dbGet().ToSQLDB(), func(tx *sql.Tx) error {
+				return busi.SagaAdjustBalance(tx, busi.TransOutUID, -req.Amount, "")
+			})
+		})
+		if err != nil {
+			return err
+		}
+		_, err = wf.NewBranch().OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			return bb.CallWithDB(dbGet().ToSQLDB(), func(tx *sql.Tx) error {
+				return busi.SagaAdjustBalance(tx, busi.TransInUID, -req.Amount, "")
+			})
+		}).NewRequest().SetBody(req).Post(Busi + "/SagaBTransIn")
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	err := workflow.Execute(gid, gid, dtmimp.MustMarshal(req))
+	assert.Error(t, err, dtmcli.ErrFailure)
+	assert.Equal(t, StatusFailed, getTransStatus(gid))
+	waitTransProcessed(gid)
+}
+
+func TestWorkflowError(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolHTTP)
+	req := busi.GenReqHTTP(30, false, false)
+	gid := dtmimp.GetFuncName()
+	busi.MainSwitch.TransOutResult.SetOnce("ERROR")
+
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.ReqHTTP
+		dtmimp.MustUnmarshal(data, &req)
+		_, err := wf.NewBranch().NewRequest().SetBody(req).Post(Busi + "/TransOut")
+		return err
+	})
+
+	err := workflow.Execute(gid, gid, dtmimp.MustMarshal(req))
+	assert.Error(t, err)
+	go waitTransProcessed(gid)
+	cronTransOnceForwardCron(t, gid, 1000)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+}
+
+func TestWorkflowOngoing(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolHTTP)
+	req := busi.GenReqHTTP(30, false, false)
+	gid := dtmimp.GetFuncName()
+	busi.MainSwitch.TransOutResult.SetOnce("ONGOING")
+
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.ReqHTTP
+		dtmimp.MustUnmarshal(data, &req)
+		_, err := wf.NewBranch().NewRequest().SetBody(req).Post(Busi + "/TransOut")
+		return err
+	})
+
+	err := workflow.Execute(gid, gid, dtmimp.MustMarshal(req))
+	assert.Error(t, err)
+	go waitTransProcessed(gid)
+	cronTransOnceForwardCron(t, gid, 1000)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+}

test/workflow_ongoing_test.go

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2021 yedf. All rights reserved.
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file.
+ */
+
+package test
+
+import (
+	"database/sql"
+	"testing"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmcli/logger"
+	"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow"
+	"github.com/dtm-labs/dtm/test/busi"
+	"github.com/stretchr/testify/assert"
+)
+
+var ongoingStep = 0
+
+func fetchOngoingStep(dest int) bool {
+	c := ongoingStep
+	logger.Debugf("ongoing step is: %d", c)
+	if c == dest {
+		ongoingStep++
+		return true
+	}
+	return false
+}
+
+func TestWorkflowSimpleResume(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolHTTP)
+	req := busi.GenReqHTTP(30, false, false)
+	gid := dtmimp.GetFuncName()
+	ongoingStep = 0
+
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		if fetchOngoingStep(0) {
+			return dtmcli.ErrOngoing
+		}
+		var req busi.ReqHTTP
+		dtmimp.MustUnmarshal(data, &req)
+		_, err := wf.NewBranch().NewRequest().SetBody(req).Post(Busi + "/TransOut")
+		return err
+	})
+
+	err := workflow.Execute(gid, gid, dtmimp.MustMarshal(req))
+	assert.Error(t, err)
+	go waitTransProcessed(gid)
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+}
+
+func TestWorkflowGrpcRollbackResume(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolGRPC)
+	gid := dtmimp.GetFuncName()
+	ongoingStep = 0
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		var req busi.BusiReq
+		dtmgimp.MustProtoUnmarshal(data, &req)
+		if fetchOngoingStep(0) {
+			return dtmcli.ErrOngoing
+		}
+		wf.NewBranch().OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			if fetchOngoingStep(4) {
+				return dtmcli.ErrOngoing
+			}
+			_, err := busi.BusiCli.TransOutRevertBSaga(wf.Context, &req)
+			return err
+		})
+		_, err := busi.BusiCli.TransOutBSaga(wf.Context, &req)
+		if fetchOngoingStep(1) {
+			return dtmcli.ErrOngoing
+		}
+		if err != nil {
+			return err
+		}
+		wf.NewBranch().OnRollback(func(bb *dtmcli.BranchBarrier) error {
+			if fetchOngoingStep(3) {
+				return dtmcli.ErrOngoing
+			}
+			_, err := busi.BusiCli.TransInRevertBSaga(wf.Context, &req)
+			return err
+		})
+		_, err = busi.BusiCli.TransInBSaga(wf.Context, &req)
+		if fetchOngoingStep(2) {
+			return dtmcli.ErrOngoing
+		}
+		return err
+	}, func(wf *workflow.Workflow) {
+		wf.Options.CompensateErrorBranch = true
+	})
+	req := &busi.ReqGrpc{Amount: 30, TransInResult: "FAILURE"}
+	err := workflow.Execute(gid, gid, dtmgimp.MustProtoMarshal(req))
+	assert.Error(t, err, dtmcli.ErrOngoing)
+	assert.Equal(t, StatusPrepared, getTransStatus(gid))
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusPrepared, getTransStatus(gid))
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusPrepared, getTransStatus(gid))
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusPrepared, getTransStatus(gid))
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusPrepared, getTransStatus(gid))
+	// next cron will make a workflow submit, and do an additional write to chan, so make an additional read chan
+	go waitTransProcessed(gid)
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusFailed, getTransStatus(gid))
+}
+
+func TestWorkflowXaResume(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolGRPC)
+	ongoingStep = 0
+	gid := dtmimp.GetFuncName()
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		_, err := wf.NewBranch().DoXa(busi.BusiConf, func(db *sql.DB) ([]byte, error) {
+			if fetchOngoingStep(0) {
+				return nil, dtmcli.ErrOngoing
+			}
+			return nil, busi.SagaAdjustBalance(db, busi.TransOutUID, -30, dtmcli.ResultSuccess)
+		})
+		if err != nil {
+			return err
+		}
+		_, err = wf.NewBranch().DoXa(busi.BusiConf, func(db *sql.DB) ([]byte, error) {
+			if fetchOngoingStep(1) {
+				return nil, dtmcli.ErrOngoing
+			}
+			return nil, busi.SagaAdjustBalance(db, busi.TransInUID, 30, dtmcli.ResultSuccess)
+		})
+		if err != nil {
+			return err
+		}
+		if fetchOngoingStep(2) {
+			return dtmcli.ErrOngoing
+		}
+
+		return err
+	})
+	err := workflow.Execute(gid, gid, nil)
+	assert.Equal(t, dtmcli.ErrOngoing, err)
+
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusPrepared, getTransStatus(gid))
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusPrepared, getTransStatus(gid))
+	// next cron will make a workflow submit, and do an additional write to chan, so make an additional read chan
+	go waitTransProcessed(gid)
+	cronTransOnceForwardNow(t, gid, 1000)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+}

test/workflow_xa_test.go

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2021 yedf. All rights reserved.
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file.
+ */
+
+package test
+
+import (
+	"database/sql"
+	"testing"
+
+	"github.com/dtm-labs/dtm/dtmcli"
+	"github.com/dtm-labs/dtm/dtmcli/dtmimp"
+	"github.com/dtm-labs/dtm/dtmcli/logger"
+	"github.com/dtm-labs/dtm/dtmgrpc/workflow"
+	"github.com/dtm-labs/dtm/test/busi"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWorkflowXaAction(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolGRPC)
+	gid := dtmimp.GetFuncName()
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		_, err := wf.NewBranch().DoXa(busi.BusiConf, func(db *sql.DB) ([]byte, error) {
+			return nil, busi.SagaAdjustBalance(db, busi.TransOutUID, -30, dtmcli.ResultSuccess)
+		})
+		if err != nil {
+			return err
+		}
+		_, err = wf.NewBranch().DoXa(busi.BusiConf, func(db *sql.DB) ([]byte, error) {
+			return nil, busi.SagaAdjustBalance(db, busi.TransInUID, 30, dtmcli.ResultSuccess)
+		})
+		return err
+	})
+	err := workflow.Execute(gid, gid, nil)
+	assert.Nil(t, err)
+	waitTransProcessed(gid)
+	assert.Equal(t, StatusSucceed, getTransStatus(gid))
+}
+
+func TestWorkflowXaRollback(t *testing.T) {
+	workflow.SetProtocolForTest(dtmimp.ProtocolGRPC)
+	gid := dtmimp.GetFuncName()
+	workflow.Register(gid, func(wf *workflow.Workflow, data []byte) error {
+		_, err := wf.NewBranch().DoXa(busi.BusiConf, func(db *sql.DB) ([]byte, error) {
+			return nil, busi.SagaAdjustBalance(db, busi.TransOutUID, -30, dtmcli.ResultSuccess)
+		})
+		if err != nil {
+			return err
+		}
+		_, err = wf.NewBranch().DoXa(busi.BusiConf, func(db *sql.DB) ([]byte, error) {
+			e := busi.SagaAdjustBalance(db, busi.TransInUID, 30, dtmcli.ResultSuccess)
+			logger.FatalIfError(e)
+			return nil, dtmcli.ErrFailure
+		})
+		return err
+	})
+	err := workflow.Execute(gid, gid, nil)
+	assert.Equal(t, dtmcli.ErrFailure, err)
+	waitTransProcessed(gid)
+	assert.Equal(t, StatusFailed, getTransStatus(gid))
+}

test/xa_cover_test.go

@@ -14,7 +14,7 @@ func TestXaCoverDBError(t *testing.T) {
 	oldDriver := busi.BusiConf.Driver
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.XaGlobalTransaction(DtmServer, gid, func(xa *dtmcli.Xa) (*resty.Response, error) {
-		req := busi.GenTransReq(30, false, false)
+		req := busi.GenReqHTTP(30, false, false)
 		_, err := xa.CallBranch(req, busi.Busi+"/TransOutXa")
 		assert.Nil(t, err)
 		busi.BusiConf.Driver = "no-driver"
@@ -44,7 +44,7 @@ func TestXaCoverGidError(t *testing.T) {
 	}
 	gid := dtmimp.GetFuncName() + "-'  '"
 	err := dtmcli.XaGlobalTransaction(DtmServer, gid, func(xa *dtmcli.Xa) (*resty.Response, error) {
-		req := busi.GenTransReq(30, false, false)
+		req := busi.GenReqHTTP(30, false, false)
 		_, err := xa.CallBranch(req, busi.Busi+"/TransOutXa")
 		assert.Error(t, err)
 		return nil, err

test/xa_grpc_test.go

@@ -21,7 +21,7 @@ import (
 func TestXaGrpcNormal(t *testing.T) {
 	gid := dtmimp.GetFuncName()
 	err := dtmgrpc.XaGlobalTransaction(DtmGrpcServer, gid, func(xa *dtmgrpc.XaGrpc) error {
-		req := busi.GenBusiReq(30, false, false)
+		req := busi.GenReqGrpc(30, false, false)
 		r := &emptypb.Empty{}
 		err := xa.CallBranch(req, busi.BusiGrpc+"/busi.Busi/TransOutXa", r)
 		if err != nil {
@@ -38,7 +38,7 @@ func TestXaGrpcNormal(t *testing.T) {
 func TestXaGrpcRollback(t *testing.T) {
 	gid := dtmimp.GetFuncName()
 	err := dtmgrpc.XaGlobalTransaction(DtmGrpcServer, gid, func(xa *dtmgrpc.XaGrpc) error {
-		req := busi.GenBusiReq(30, false, true)
+		req := busi.GenReqGrpc(30, false, true)
 		r := &emptypb.Empty{}
 		err := xa.CallBranch(req, busi.BusiGrpc+"/busi.Busi/TransOutXa", r)
 		if err != nil {

test/xa_test.go

@@ -21,7 +21,7 @@ import (
 func TestXaNormal(t *testing.T) {
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.XaGlobalTransaction(dtmutil.DefaultHTTPServer, gid, func(xa *dtmcli.Xa) (*resty.Response, error) {
-		req := busi.GenTransReq(30, false, false)
+		req := busi.GenReqHTTP(30, false, false)
 		resp, err := xa.CallBranch(req, busi.Busi+"/TransOutXa")
 		if err != nil {
 			return resp, err
@@ -37,7 +37,7 @@ func TestXaNormal(t *testing.T) {
 func TestXaDuplicate(t *testing.T) {
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.XaGlobalTransaction(DtmServer, gid, func(xa *dtmcli.Xa) (*resty.Response, error) {
-		req := busi.GenTransReq(30, false, false)
+		req := busi.GenReqHTTP(30, false, false)
 		_, err := xa.CallBranch(req, busi.Busi+"/TransOutXa")
 		assert.Nil(t, err)
 		sdb, err := dtmimp.StandaloneDB(busi.BusiConf)
@@ -59,7 +59,7 @@ func TestXaDuplicate(t *testing.T) {
 func TestXaRollback(t *testing.T) {
 	gid := dtmimp.GetFuncName()
 	err := dtmcli.XaGlobalTransaction(DtmServer, gid, func(xa *dtmcli.Xa) (*resty.Response, error) {
-		req := busi.GenTransReq(30, false, true)
+		req := busi.GenReqHTTP(30, false, true)
 		resp, err := xa.CallBranch(req, busi.Busi+"/TransOutXa")
 		if err != nil {
 			return resp, err
@@ -107,7 +107,7 @@ func TestXaNotTimeout(t *testing.T) {
 			timeoutChan <- 0
 		}()
 		<-timeoutChan
-		req := busi.GenTransReq(30, false, false)
+		req := busi.GenReqHTTP(30, false, false)
 		_, err := xa.CallBranch(req, busi.Busi+"/TransOutXa")
 		assert.Nil(t, err)
 		busi.MainSwitch.NextResult.SetOnce(dtmcli.ResultOngoing) // make commit temp error