Browse Source

Merge branch 'main' into main

pull/184/head
yedf2 4 years ago
committed by GitHub
parent
commit
68efe31a3e
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 70
      .github/workflows/codeql-analysis.yml
  2. 48
      .github/workflows/tests.yml
  3. 10
      .gitignore
  4. 2
      .golangci.yml
  5. 18
      .travis.yml
  6. 33
      .vscode/launch.json.sample
  7. 3
      .vscode/settings.json.sample
  8. 128
      README-cn.md
  9. 14
      README-en.md
  10. 180
      README.md
  11. 82
      app/main.go
  12. 28
      bench/Makefile
  13. 63
      bench/main.go
  14. 10
      bench/prepare.sh
  15. 8
      bench/run-mysql.sh
  16. 5
      bench/run-services.sh
  17. 6
      bench/setup-redis6.sh
  18. 5
      bench/setup.sh
  19. 118
      bench/svr/http.go
  20. 5
      bench/test-boltdb.sh
  21. 13
      bench/test-mysql.sh
  22. 27
      bench/test-redis.sh
  23. 120
      common/config.go
  24. 49
      common/config_test.go
  25. 30
      common/types.go
  26. 66
      common/types_test.go
  27. 121
      common/utils.go
  28. 48
      conf.sample.yml
  29. 81
      dtmcli/barrier.go
  30. 2
      dtmcli/consts.go
  31. 2
      dtmcli/dtmimp/README.md
  32. 31
      dtmcli/dtmimp/trans_base.go
  33. 12
      dtmcli/dtmimp/trans_xa_base.go
  34. 3
      dtmcli/dtmimp/types.go
  35. 1
      dtmcli/dtmimp/types_test.go
  36. 133
      dtmcli/dtmimp/utils.go
  37. 18
      dtmcli/dtmimp/utils_test.go
  38. 18
      dtmcli/dtmimp/vars.go
  39. 111
      dtmcli/logger/log.go
  40. 52
      dtmcli/logger/logger_test.go
  41. 26
      dtmcli/msg.go
  42. 2
      dtmcli/saga.go
  43. 10
      dtmcli/tcc.go
  44. 44
      dtmcli/types.go
  45. 7
      dtmcli/types_test.go
  46. 18
      dtmcli/xa.go
  47. 4
      dtmgrpc/barrier.go
  48. 2
      dtmgrpc/dtmgimp/README.md
  49. 510
      dtmgrpc/dtmgimp/dtmgimp.pb.go
  50. 33
      dtmgrpc/dtmgimp/grpc_clients.go
  51. 40
      dtmgrpc/dtmgimp/types.go
  52. 55
      dtmgrpc/dtmgimp/utils.go
  53. 534
      dtmgrpc/dtmgpb/dtmgimp.pb.go
  54. 4
      dtmgrpc/dtmgpb/dtmgimp.proto
  55. 10
      dtmgrpc/dtmgpb/dtmgimp_grpc.pb.go
  56. 26
      dtmgrpc/msg.go
  57. 4
      dtmgrpc/saga.go
  58. 46
      dtmgrpc/tcc.go
  59. 38
      dtmgrpc/type.go
  60. 5
      dtmgrpc/type_test.go
  61. 25
      dtmgrpc/xa.go
  62. 47
      dtmsvr/api.go
  63. 26
      dtmsvr/api_grpc.go
  64. 46
      dtmsvr/api_http.go
  65. 107
      dtmsvr/config/config.go
  66. 84
      dtmsvr/config/config_test.go
  67. 43
      dtmsvr/config/config_utils.go
  68. 20
      dtmsvr/cron.go
  69. 165
      dtmsvr/storage/boltdb/boltdb.go
  70. 10
      dtmsvr/storage/boltdb/boltdb_test.go
  71. 257
      dtmsvr/storage/redis.go
  72. 300
      dtmsvr/storage/redis/redis.go
  73. 25
      dtmsvr/storage/registry/factory.go
  74. 45
      dtmsvr/storage/registry/registry.go
  75. 140
      dtmsvr/storage/sql.go
  76. 178
      dtmsvr/storage/sql/sql.go
  77. 24
      dtmsvr/storage/store.go
  78. 35
      dtmsvr/storage/trans.go
  79. 16
      dtmsvr/storage/utils.go
  80. 73
      dtmsvr/svr.go
  81. 7
      dtmsvr/svr_imports.go
  82. 44
      dtmsvr/trans_class.go
  83. 68
      dtmsvr/trans_process.go
  84. 68
      dtmsvr/trans_status.go
  85. 16
      dtmsvr/trans_type_msg.go
  86. 120
      dtmsvr/trans_type_saga.go
  87. 7
      dtmsvr/trans_type_tcc.go
  88. 4
      dtmsvr/trans_type_xa.go
  89. 22
      dtmsvr/utils.go
  90. 4
      dtmsvr/utils_test.go
  91. 14
      dtmutil/consts.go
  92. 32
      dtmutil/db.go
  93. 157
      dtmutil/utils.go
  94. 18
      dtmutil/utils_test.go
  95. 14
      examples/base.go
  96. 192
      examples/base_http.go
  97. 107
      examples/base_types.go
  98. 330
      examples/busi.pb.go
  99. 55
      examples/data.go
  100. 25
      examples/grpc_msg.go

70
.github/workflows/codeql-analysis.yml

@ -0,0 +1,70 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"
on:
push:
branches: [ main ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ main ]
schedule:
- cron: '25 19 * * 2'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'go' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1

48
.github/workflows/tests.yml

@ -0,0 +1,48 @@
name: Tests
on:
push:
branches-ignore:
- 'tmp-*'
pull_request:
branches-ignore:
- 'tmp-*'
jobs:
tests:
name: CI
runs-on: ubuntu-latest
services:
mysql:
image: 'mysql:5.7'
env:
MYSQL_ALLOW_EMPTY_PASSWORD: 1
volumes:
- /etc/localtime:/etc/localtime:ro
- /etc/timezone:/etc/timezone:ro
ports:
- 3306:3306
reids:
image: 'redis'
volumes:
- /etc/localtime:/etc/localtime:ro
- /etc/timezone:/etc/timezone:ro
ports:
- 6379:6379
steps:
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Check out code
uses: actions/checkout@v2
- name: Install dependencies
run: |
go mod download
- name: Run CI lint
run: make lint
- name: Run test cover
run: sh helper/test-cover.sh

10
.gitignore

@ -5,10 +5,14 @@ conf.yml
main
dist
.idea/**
.vscode/*.json
.vscode
default.etcd
*/**/*.bolt
# Output file of unit test coverage
coverage.out
coverage.out.tmp
coverage.*
profile.*
test.sh
dtm
dtm-*
dtm.*

2
.golangci.yml

@ -2,7 +2,7 @@ run:
deadline: 5m
skip-dirs:
- test
- examples
# - bench
linter-settings:
goconst:

18
.travis.yml

@ -1,18 +0,0 @@
language: go
go:
- 1.15.x
env:
- GO111MODULE=on
branches:
only:
- master
- main
- alpha
services:
- mysql
- redis-server
before_install:
- go get -t -v ./...
- go get github.com/yedf2/goveralls
script:
- $GOPATH/bin/goveralls -envs=TEST_STORE=redis,TEST_STORE=mysql,TEST_STORE=boltdb -flags '-gcflags=-l' -service=travis-ci -ignore="examples/*,dtmgrpc/dtmgimp/*.pb.go,bench/*,test/*"

33
.vscode/launch.json.sample

@ -1,33 +0,0 @@
{
// 使用 IntelliSense 了解相关属性。
// 悬停以查看现有属性的描述。
// 欲了解更多信息,请访问: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/app/main.go",
"cwd": "${workspaceFolder}",
"env": {
// "GIN_MODE": "release"
},
"args": ["grpc_saga"]
},
{
"name": "Test",
"type": "go",
"request": "launch",
"mode": "test",
"port": 2345,
"host": "127.0.0.1",
"program": "${file}",
"env": {
// "GIN_MODE": "release"
},
"args": []
}
]
}

3
.vscode/settings.json.sample

@ -1,3 +0,0 @@
{
"go.formatTool": "gofmt"
}

128
README-cn.md

@ -1,8 +1,8 @@
![license](https://img.shields.io/github/license/yedf/dtm)
[![Build Status](https://travis-ci.com/yedf/dtm.svg?branch=main)](https://travis-ci.com/yedf/dtm)
[![Coverage Status](https://coveralls.io/repos/github/yedf/dtm/badge.svg?branch=main)](https://coveralls.io/github/yedf/dtm?branch=main)
[![Go Report Card](https://goreportcard.com/badge/github.com/yedf/dtm)](https://goreportcard.com/report/github.com/yedf/dtm)
[![Go Reference](https://pkg.go.dev/badge/github.com/yedf/dtm.svg)](https://pkg.go.dev/github.com/yedf/dtm)
![license](https://img.shields.io/github/license/dtm-labs/dtm)
![Build Status](https://github.com/dtm-labs/dtm/actions/workflows/tests.yml/badge.svg?branch=main)
[![codecov](https://codecov.io/gh/dtm-labs/dtm/branch/main/graph/badge.svg?token=UKKEYQLP3F)](https://codecov.io/gh/dtm-labs/dtm)
[![Go Report Card](https://goreportcard.com/badge/github.com/dtm-labs/dtm)](https://goreportcard.com/report/github.com/dtm-labs/dtm)
[![Go Reference](https://pkg.go.dev/badge/github.com/dtm-labs/dtm.svg)](https://pkg.go.dev/github.com/dtm-labs/dtm)
[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go#database)
# [English Docs](https://en.dtm.pub)
@ -12,87 +12,78 @@ DTM是一款golang开发的分布式事务管理器,解决了跨数据库、
他优雅的解决了幂等、空补偿、悬挂等分布式事务难题,提供了简单易用、高性能、易水平扩展的解决方案。
作者受邀参加中国数据库大会分享[多语言环境下分布式事务实践](http://dtcc.it168.com/yicheng.html#b9)
## 谁在使用DTM(仅列出部分)
[Tencent 腾讯](https://dtm.pub/other/using.html#tencent)
## 谁在使用dtm
[Tencent 腾讯](https://www.tencent.com/)
[Ivydad 常青藤爸爸](https://dtm.pub/other/using.html#ivydad)
[Ivydad 常青藤爸爸](https://ivydad.com)
[Eglass 视咖镜小二](https://dtm.pub/other/using.html)
[Eglass 视咖镜小二](https://epeijing.cn)
[极欧科技](http://jiou.me)
[金数智联]()
[极欧科技](https://dtm.pub/other/using.html)
## 亮点
* 极易接入
- 支持HTTP,提供非常简单的接口,极大降低上手分布式事务的难度,新手也能快速接入
* 使用简单
- 开发者不再担心悬挂、空补偿、幂等各类问题,框架层代为处理
- 零配置启动服务,提供非常简单的HTTP接口,极大降低上手分布式事务的难度,新手也能快速接入
* 跨语言
- 可适合多语言栈的公司使用。方便go、python、php、nodejs、ruby、c# 各类语言使用。
* 使用简单
- 开发者不再担心悬挂、空补偿、幂等各类问题,首创子事务屏障技术代为处理
* 易部署、易扩展
- 仅依赖mysql,部署简单,易集群化,易水平扩展
- 依赖mysql|redis,部署简单,易集群化,易水平扩展
* 多种分布式事务协议支持
- TCC、SAGA、XA、事务消息
- TCC、SAGA、XA、二阶段消息,一站式解决所有分布式事务问题
## 与其他框架对比
目前开源的分布式事务框架,Java的框架较多,有大厂开源的SEATA、ServiceComb-Pack,shardingsphere,以及个人开源的himly,tcc-transaction,ByteTCC等等,其中以Seata的应用最为广泛。
非Java语言类的,暂未看到除dtm之外的成熟框架,因此这里仅将DTM和Java中最成熟的Seata对比:
非Java语言类的,暂未看到除dtm之外的成熟框架,因此这里将DTM和Java中最成熟的Seata对比:
| 特性| DTM | SEATA |备注|
|:-----:|:----:|:----:|:----:|
| 支持语言 |<span style="color:green">Go、Java、python、php、c#...</span>|<span style="color:orange">Java</span>|dtm可轻松接入一门新语言|
|异常处理| <span style="color:green"> [子事务屏障自动处理](https://zhuanlan.zhihu.com/p/388444465) </span>|<span style="color:orange">手动处理</span> |dtm解决了幂等、悬挂、空补偿|
| TCC事务| <span style="color:green"></span>|<span style="color:green"></span>||
| XA事务|<span style="color:green"></span>|<span style="color:green"></span>||
|AT事务|<span style="color:orange">建议使用XA</span>|<span style="color:green"></span>|AT与XA类似,性能更好,但有脏回滚|
| SAGA事务 |<span style="color:green">支持并发</span> |<span style="color:green">状态机模式</span> ||
|事务消息|<span style="color:green"></span>|<span style="color:red"></span>|dtm提供类似rocketmq的事务消息|
|单服务多数据源|<span style="color:green"></span>|<span style="color:red"></span>||
|通信协议|HTTP、gRPC、go-zero|dubbo等协议|dtm对云原生更加友好|
|star数量|<img src="https://img.shields.io/github/stars/yedf/dtm.svg?style=social" alt="github stars"/>|<img src="https://img.shields.io/github/stars/seata/seata.svg?style=social" alt="github stars"/>|dtm从20210604发布0.1,发展快|
| [支持语言](https://dtm.pub/other/opensource.html#lang) |<span style="color:green">Go、Java、python、php、c#...</span>|<span style="color:orange">Java</span>|dtm可轻松接入一门新语言|
|[异常处理](https://dtm.pub/other/opensource.html#exception)| <span style="color:green"> 子事务屏障自动处理 </span>|<span style="color:orange">手动处理</span> |dtm解决了幂等、悬挂、空补偿|
| [TCC事务](https://dtm.pub/other/opensource.html#tcc)| <span style="color:green"></span>|<span style="color:green"></span>||
| [XA事务](https://dtm.pub/other/opensource.html#xa)|<span style="color:green"></span>|<span style="color:green"></span>||
|[AT事务](https://dtm.pub/other/opensource.html#at)|<span style="color:orange">建议使用XA</span>|<span style="color:green"></span>|AT与XA类似,性能更好,但有脏回滚|
| [SAGA事务](https://dtm.pub/other/opensource.html#saga) |<span style="color:green">支持并发</span> |<span style="color:green">状态机模式</span> ||
|[二阶段消息](https://dtm.pub/other/opensource.html#msg)|<span style="color:green"></span>|<span style="color:red"></span>|dtm提供类似rocketmq的事务消息|
|[单服务多数据源](https://dtm.pub/other/opensource.html#multidb)|<span style="color:green"></span>|<span style="color:red"></span>||
|[通信协议](https://dtm.pub/other/opensource.html#protocol)|HTTP、gRPC、go-zero|dubbo等协议|dtm对云原生更加友好|
|[star数量](https://dtm.pub/other/opensource.html#star)|<img src="https://img.shields.io/github/stars/dtm-labs/dtm.svg?style=social" alt="github stars"/>|<img src="https://img.shields.io/github/stars/seata/seata.svg?style=social" alt="github stars"/>|dtm从20210604发布0.1,发展快|
从上面对比的特性来看,如果您的语言栈包含了Java之外的语言,那么dtm是您的首选。如果您的语言栈是Java,您也可以选择接入dtm,使用子事务屏障技术,简化您的业务编写。
详细的对比可以点击特性中的链接,跳到相关文档
## [性能测试报告](https://dtm.pub/other/performance.html)
## [教程与文档](https://dtm.pub)
## [各语言客户端及示例](https://dtm.pub/summary/code.html#go)
## [各语言客户端及示例](https://dtm.pub/ref/sdk.html#go)
## 微服务框架支持
- [go-zero](https://github.com/zeromicro/go-zero):一开源就非常火爆的微服务框架,首家接入dtm的微服务框架。感谢go-zero作者[kevwan](https://github.com/kevwan)的大力支持
- [polaris](https://github.com/polarismesh/polaris): 腾讯开源的注册发现组件,以及在其上构建的微服务框架。感谢腾讯同学[ychensha](https://github.com/ychensha)的PR
- 其他:看用户需求量,择机接入
- 其他:看用户需求量,择机接入,参见[微服务支持](https://dtm.pub/ref/proto.html)
具体微服务接入使用,参见[微服务支持](https://dtm.pub/protocol/intro.html)
## 快速开始
### 获取代码
`git clone https://github.com/yedf/dtm && cd dtm`
如果您不是Go语言,可以跳转[各语言客户端及示例](https://dtm.pub/ref/sdk.html#go),里面有相关的快速开始示例
### dtm依赖于mysql
### 运行dtm
安装[docker 20.04+](https://docs.docker.com/get-docker/)之后
`docker-compose -f helper/compose.mysql.yml up`
``` bash
git clone https://github.com/dtm-labs/dtm && cd dtm
go run main.go
```
> 您也可以配置使用现有的mysql,需要高级权限,允许dtm创建数据库
>
> `cp conf.sample.yml conf.yml # 修改conf.yml`
### 启动并运行一个saga示例
下面运行一个类似跨行转账的示例,包括两个事务分支:资金转出(TransOut)、资金转入(TransIn)。DTM保证TransIn和TransOut要么全部成功,要么全部回滚,保证最终金额的正确性。
### 启动并运行saga示例
`go run app/main.go qs`
`go run qs/main.go`
## 开始使用
## 接入详解
### 使用
### 接入代码
``` GO
// 具体业务微服务地址
const qsBusi = "http://localhost:8081/api/busi_saga"
@ -116,23 +107,34 @@ DTM是一款golang开发的分布式事务管理器,解决了跨数据库、
<img src="https://pic3.zhimg.com/80/v2-b7d98659093c399e182a0173a8e549ca_1440w.jpg" height=428 />
### 完整示例
参考[examples/quick_start.go](./examples/quick_start.go)
### 失败情况
在实际的业务中,子事务可能出现失败,例如转入的子账号被冻结导致转账失败。我们对业务代码进行修改,让TransIn的正向操作失败,然后看看结果
## 公众号
您可以关注公众号:分布式事务,及时跟踪dtm的最新内容
## 交流群
``` go
app.POST(qsBusiAPI+"/TransIn", common.WrapHandler2(func(c *gin.Context) interface{} {
return dtmcli.ErrFailure
}))
```
再运行这个例子,整个事务最终失败,时序图如下:
<img src="https://pic3.zhimg.com/80/v2-8d8f1476be8a1e2e09ce97a89b4116c2_1440w.jpg" height=528 />
在转入操作失败的情况下,TransIn和TransOut的补偿操作被执行,保证了最终的余额和转账前是一样的。
### 更多示例
参考[dtm-labs/dtm-examples](https://github.com/dtm-labs/dtm-examples)
## 联系我们
### 公众号
dtm官方公众号:分布式事务,大量干货分享,以及dtm的最新消息
### 交流群
请加 yedf2008 好友或者扫码加好友,验证回复 dtm 按照指引进群
![yedf2008](http://service.ivydad.com/cover/dubbingb6b5e2c0-2d2a-cd59-f7c5-c6b90aceb6f1.jpeg)
欢迎使用[dtm](https://github.com/yedf/dtm),或者通过dtm学习实践分布式事务相关知识,欢迎star支持我们
### github
作者github: [https://github.com/yedf2](https://github.com/yedf2)
欢迎使用[dtm](https://github.com/dtm-labs/dtm),或者通过dtm学习实践分布式事务相关知识,欢迎star支持我们
## 谁在使用
<div style='vertical-align: middle'>
<img alt='腾讯' height='80' src='https://dtm.pub/assets/tencent.4b87bfd8.jpeg' /img>
<img alt='常青藤爸爸' height='80' src='https://dtm.pub/assets/ivydad.d0f58a94.png' /img>
<img alt='镜小二' height='80' src='https://img.epeijing.cn/official-website/assets/logo.png' /img>
<img alt='极欧科技' height='80' src='https://dtm.pub/assets/jiou.5bed10c2.png' /img>
<img alt='金数智联' height='80' src='https://dtm.pub/assets/gdci.214d305a.png' /img>
</div>

14
README-en.md

@ -1,8 +1,8 @@
![license](https://img.shields.io/github/license/yedf/dtm)
[![Build Status](https://travis-ci.com/yedf/dtm.svg?branch=main)](https://travis-ci.com/yedf/dtm)
[![Coverage Status](https://coveralls.io/repos/github/yedf/dtm/badge.svg?branch=main)](https://coveralls.io/github/yedf/dtm?branch=main)
[![Go Report Card](https://goreportcard.com/badge/github.com/yedf/dtm)](https://goreportcard.com/report/github.com/yedf/dtm)
[![Go Reference](https://pkg.go.dev/badge/github.com/yedf/dtm.svg)](https://pkg.go.dev/github.com/yedf/dtm)
![license](https://img.shields.io/github/license/dtm-labs/dtm)
![Build Status](https://github.com/dtm-labs/dtm/actions/workflows/tests.yml/badge.svg?branch=main)
[![codecov](https://codecov.io/gh/dtm-labs/dtm/branch/main/graph/badge.svg?token=UKKEYQLP3F)](https://codecov.io/gh/dtm-labs/dtm)
[![Go Report Card](https://goreportcard.com/badge/github.com/dtm-labs/dtm)](https://goreportcard.com/report/github.com/dtm-labs/dtm)
[![Go Reference](https://pkg.go.dev/badge/github.com/dtm-labs/dtm.svg)](https://pkg.go.dev/github.com/dtm-labs/dtm)
[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go#database)
# [中文文档](http://dtm.pub)
@ -62,7 +62,7 @@ The following is a comparison of the main features of dtm and Seata.
| Transactional Messaging | <span style="color:green"></span> | <span style="color:red"></span> | dtm provides Transactional Messaging similar to RocketMQ |
| Multiple DBs in a service |<span style="color:green"></span>|<span style="color:red"></span>||
| Communication protocols | <span style="color:green">HTTP, gRPC</span> | <span style="color:green">Dubbo, no HTTP</span> | |
| Star count | <img src="https://img.shields.io/github/stars/yedf/dtm.svg?style=social" alt="github stars"/> | <img src="https://img.shields.io/github/stars/seata/seata.svg?style=social" alt="github stars"/> | dtm 0.1 is released from 20210604 and under fast development |
| Star count | <img src="https://img.shields.io/github/stars/dtm-labs/dtm.svg?style=social" alt="github stars"/> | <img src="https://img.shields.io/github/stars/seata/seata.svg?style=social" alt="github stars"/> | dtm 0.1 is released from 20210604 and under fast development |
From the features' comparison above, if your language stack includes languages other than Java, then dtm is the one for you.
If your language stack is Java, you can also choose to access dtm and use sub-transaction barrier technology to simplify your business development.
@ -73,7 +73,7 @@ If your language stack is Java, you can also choose to access dtm and use sub-tr
### Install
`git clone https://github.com/yedf/dtm`
`git clone https://github.com/dtm-labs/dtm`
### Configure Mysql

180
README.md

@ -1,128 +1,140 @@
![license](https://img.shields.io/github/license/yedf/dtm)
[![Build Status](https://travis-ci.com/yedf/dtm.svg?branch=main)](https://travis-ci.com/yedf/dtm)
[![Coverage Status](https://coveralls.io/repos/github/yedf/dtm/badge.svg?branch=main)](https://coveralls.io/github/yedf/dtm?branch=main)
[![Go Report Card](https://goreportcard.com/badge/github.com/yedf/dtm)](https://goreportcard.com/report/github.com/yedf/dtm)
[![Go Reference](https://pkg.go.dev/badge/github.com/yedf/dtm.svg)](https://pkg.go.dev/github.com/yedf/dtm)
![license](https://img.shields.io/github/license/dtm-labs/dtm)
![Build Status](https://github.com/dtm-labs/dtm/actions/workflows/tests.yml/badge.svg?branch=main)
[![codecov](https://codecov.io/gh/dtm-labs/dtm/branch/main/graph/badge.svg?token=UKKEYQLP3F)](https://codecov.io/gh/dtm-labs/dtm)
[![Go Report Card](https://goreportcard.com/badge/github.com/dtm-labs/dtm)](https://goreportcard.com/report/github.com/dtm-labs/dtm)
[![Go Reference](https://pkg.go.dev/badge/github.com/dtm-labs/dtm.svg)](https://pkg.go.dev/github.com/dtm-labs/dtm)
[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go#database)
# [中文文档](http://dtm.pub)
# [English Docs](https://en.dtm.pub)
# 跨语言分布式事务管理器
# A Cross Language Distributed Transaction Manager
DTM是一款golang开发的分布式事务管理器,解决了跨数据库、跨服务、跨语言栈更新数据的一致性问题。
## Who's using DTM
他优雅的解决了幂等、空补偿、悬挂等分布式事务难题,提供了简单易用、高性能、易水平扩展的解决方案。
[Tencent](https://www.tencent.com/)
## 谁在使用DTM(仅列出部分)
[Tencent 腾讯](https://dtm.pub/other/using.html#tencent)
[Ivydad](https://ivydad.com)
[Ivydad 常青藤爸爸](https://dtm.pub/other/using.html#ivydad)
[Eglass](https://epeijing.cn)
[Eglass 视咖镜小二](https://dtm.pub/other/using.html)
[Jiou](http://jiou.me)
[极欧科技](https://dtm.pub/other/using.html)
[GoldenData]()
## 亮点
## What is DTM
* 极易接入
- 零配置启动服务,提供非常简单的HTTP接口,极大降低上手分布式事务的难度,新手也能快速接入
* 跨语言
- 可适合多语言栈的公司使用。方便go、python、php、nodejs、ruby、c# 各类语言使用。
* 使用简单
- 开发者不再担心悬挂、空补偿、幂等各类问题,首创子事务屏障技术代为处理
* 易部署、易扩展
- 依赖mysql|redis,部署简单,易集群化,易水平扩展
* 多种分布式事务协议支持
- TCC、SAGA、XA、二阶段消息,一站式解决所有分布式事务问题
DTM is the first distributed transaction management framework in Golang. Unlike other frameworks, DTM provides extremely easy access interfaces of HTTP and gRPC, supports multiple language bindings, and handles tricky problems of unordered sub-transactions at the framework level.
## 与其他框架对比
## Features
非Java语言类的,暂未看到除dtm之外的成熟框架,因此这里将DTM和Java中最成熟的Seata对比:
* Extremely easy to adapt
- Support HTTP and gRPC, provide easy-to-use programming interfaces, lower substantially the barrier of getting started with distributed transactions. Newcomers can adapt quickly.
| 特性| DTM | SEATA |备注|
|:-----:|:----:|:----:|:----:|
| [支持语言](https://dtm.pub/other/opensource.html#lang) |<span style="color:green">Go、Java、python、php、c#...</span>|<span style="color:orange">Java</span>|dtm可轻松接入一门新语言|
|[异常处理](https://dtm.pub/other/opensource.html#exception)| <span style="color:green"> 子事务屏障自动处理 </span>|<span style="color:orange">手动处理</span> |dtm解决了幂等、悬挂、空补偿|
| [TCC事务](https://dtm.pub/other/opensource.html#tcc)| <span style="color:green"></span>|<span style="color:green"></span>||
| [XA事务](https://dtm.pub/other/opensource.html#xa)|<span style="color:green"></span>|<span style="color:green"></span>||
|[AT事务](https://dtm.pub/other/opensource.html#at)|<span style="color:orange">建议使用XA</span>|<span style="color:green"></span>|AT与XA类似,性能更好,但有脏回滚|
| [SAGA事务](https://dtm.pub/other/opensource.html#saga) |<span style="color:green">支持并发</span> |<span style="color:green">状态机模式</span> ||
|[二阶段消息](https://dtm.pub/other/opensource.html#msg)|<span style="color:green"></span>|<span style="color:red"></span>|dtm提供类似rocketmq的事务消息|
|[单服务多数据源](https://dtm.pub/other/opensource.html#multidb)|<span style="color:green"></span>|<span style="color:red"></span>||
|[通信协议](https://dtm.pub/other/opensource.html#protocol)|HTTP、gRPC、go-zero|dubbo等协议|dtm对云原生更加友好|
|[star数量](https://dtm.pub/other/opensource.html#star)|<img src="https://img.shields.io/github/stars/dtm-labs/dtm.svg?style=social" alt="github stars"/>|<img src="https://img.shields.io/github/stars/seata/seata.svg?style=social" alt="github stars"/>|dtm从20210604发布0.1,发展快|
* Easy to use
- Relieving developers from worrying about suspension, null compensation, idempotent transaction, and other tricky problems, the framework layer handles them all.
从上面对比的特性来看,如果您的语言栈包含了Java之外的语言,那么dtm是您的首选。如果您的语言栈是Java,您也可以选择接入dtm,使用子事务屏障技术,简化您的业务编写。
* Language-agnostic
- Suit for companies with multiple-language stacks.
Easy to write bindings for Go, Python, PHP, Node.js, Ruby, and other languages.
详细的对比可以点击特性中的链接,跳到相关文档
## [性能测试报告](https://dtm.pub/other/performance.html)
* Easy to deploy, easy to extend
- DTM depends only on MySQL, easy to deploy, cluster, and scale horizontally.
## [教程与文档](https://dtm.pub)
* Support for multiple distributed transaction protocol
- TCC, SAGA, XA, Transactional messages.
## [各语言客户端及示例](https://dtm.pub/ref/sdk.html#go)
## DTM vs. others
## 微服务框架支持
- [go-zero](https://github.com/zeromicro/go-zero):一开源就非常火爆的微服务框架,首家接入dtm的微服务框架。感谢go-zero作者[kevwan](https://github.com/kevwan)的大力支持
- [polaris](https://github.com/polarismesh/polaris): 腾讯开源的注册发现组件,以及在其上构建的微服务框架。感谢腾讯同学[ychensha](https://github.com/ychensha)的PR
- 其他:看用户需求量,择机接入,参见[微服务支持](https://dtm.pub/ref/proto.html)
There is no mature open-source distributed transaction framework for non-Java languages.
Mature open-source distributed transaction frameworks for Java language include Ali's Seata, Huawei's ServiceComb-Pack, Jingdong's shardingsphere, himly, tcc-transaction, ByteTCC, and so on, of which Seata is most widely used.
## 快速开始
The following is a comparison of the main features of dtm and Seata.
如果您不是Go语言,可以跳转[各语言客户端及示例](https://dtm.pub/ref/sdk.html#go),里面有相关的快速开始示例
### 运行dtm
| Features | DTM | Seata | Remarks |
| :-----: | :----: | :----: | :----: |
| Supported languages | <span style="color:green">Golang, Python, PHP, and others</span> | <span style="color:orange">Java</span> | dtm allows easy access from a new language |
| Exception handling | [Sub-transaction barrier](https://zhuanlan.zhihu.com/p/388444465) | <span style="color:orange">manual</span> | dtm solves idempotent transaction, hanging, null compensation |
| TCC | <span style="color:green"></span> | <span style="color:green"></span> | |
| XA | <span style="color:green"></span> | <span style="color:green"></span> | |
| AT | <span style="color:orange">suggest XA</span> | <span style="color:green"></span> | AT is similar to XA with better performance but with dirty rollback |
| SAGA | <span style="color:green">support concurrency</span> | <span style="color:green">complicated state-machine mode</span> | dtm's state-machine mode is being planned |
| Transactional Messaging | <span style="color:green"></span> | <span style="color:red"></span> | dtm provides Transactional Messaging similar to RocketMQ |
| Multiple DBs in a service |<span style="color:green"></span>|<span style="color:red"></span>||
| Communication protocols | <span style="color:green">HTTP, gRPC</span> | <span style="color:green">Dubbo, no HTTP</span> | |
| Star count | <img src="https://img.shields.io/github/stars/yedf/dtm.svg?style=social" alt="github stars"/> | <img src="https://img.shields.io/github/stars/seata/seata.svg?style=social" alt="github stars"/> | dtm 0.1 is released from 20210604 and under fast development |
``` bash
git clone https://github.com/dtm-labs/dtm && cd dtm
go run main.go
```
From the features' comparison above, if your language stack includes languages other than Java, then dtm is the one for you.
If your language stack is Java, you can also choose to access dtm and use sub-transaction barrier technology to simplify your business development.
### 启动并运行一个saga示例
下面运行一个类似跨行转账的示例,包括两个事务分支:资金转出(TransOut)、资金转入(TransIn)。DTM保证TransIn和TransOut要么全部成功,要么全部回滚,保证最终金额的正确性。
## [Cook Book](https://en.dtm.pub)
`go run qs/main.go`
# Quick start
## 接入详解
### Install
### 接入代码
``` GO
// 具体业务微服务地址
const qsBusi = "http://localhost:8081/api/busi_saga"
req := &gin.H{"amount": 30} // 微服务的载荷
// DtmServer为DTM服务的地址,是一个url
DtmServer := "http://localhost:36789/api/dtmsvr"
saga := dtmcli.NewSaga(DtmServer, dtmcli.MustGenGid(DtmServer)).
// 添加一个TransOut的子事务,正向操作为url: qsBusi+"/TransOut", 补偿操作为url: qsBusi+"/TransOutCompensate"
Add(qsBusi+"/TransOut", qsBusi+"/TransOutCompensate", req).
// 添加一个TransIn的子事务,正向操作为url: qsBusi+"/TransIn", 补偿操作为url: qsBusi+"/TransInCompensate"
Add(qsBusi+"/TransIn", qsBusi+"/TransInCompensate", req)
// 提交saga事务,dtm会完成所有的子事务/回滚所有的子事务
err := saga.Submit()
```
`git clone https://github.com/yedf/dtm`
成功运行后,可以看到TransOut、TransIn依次被调用,完成了整个分布式事务
### Configure Mysql
### 时序图
`cp conf.sample.yml conf.yml # Modify conf.yml`
上述saga分布式事务的时序图如下:
### Start the example
`go run app/main.go`
<img src="https://pic3.zhimg.com/80/v2-b7d98659093c399e182a0173a8e549ca_1440w.jpg" height=428 />
# Code
### 失败情况
在实际的业务中,子事务可能出现失败,例如转入的子账号被冻结导致转账失败。我们对业务代码进行修改,让TransIn的正向操作失败,然后看看结果
### Use
``` go
// business micro-service address
const qsBusi = "http://localhost:8081/api/busi_saga"
// The address where DtmServer serves DTM, which is a url
DtmServer := "http://localhost:36789/api/dtmsvr"
req := &gin.H{"amount": 30} // micro-service payload
// DtmServer is the address of DTM micro-service
saga := dtmcli.NewSaga(DtmServer, dtmcli.MustGenGid(DtmServer)).
// add a TransOut subtraction,forward operation with url: qsBusi+"/TransOut", reverse compensation operation with url: qsBusi+"/TransOutCompensate"
Add(qsBusi+"/TransOut", qsBusi+"/TransOutCompensate", req).
// add a TransIn subtraction, forward operation with url: qsBusi+"/TransIn", reverse compensation operation with url: qsBusi+"/TransInCompensate"
Add(qsBusi+"/TransIn", qsBusi+"/TransInCompensate", req)
// submit the created saga transaction,dtm ensures all subtractions either complete or get revoked
err := saga.Submit()
app.POST(qsBusiAPI+"/TransIn", common.WrapHandler2(func(c *gin.Context) interface{} {
return dtmcli.ErrFailure
}))
```
### Complete example
Refer to [examples/quick_start.go](./examples/quick_start.go).
再运行这个例子,整个事务最终失败,时序图如下:
### Slack
<img src="https://pic3.zhimg.com/80/v2-8d8f1476be8a1e2e09ce97a89b4116c2_1440w.jpg" height=528 />
You can join the [DTM slack channel here](https://join.slack.com/t/dtm-w6k9662/shared_invite/zt-vkrph4k1-eFqEFnMkbmlXqfUo5GWHWw).
在转入操作失败的情况下,TransIn和TransOut的补偿操作被执行,保证了最终的余额和转账前是一样的。
### Wechat
### 更多示例
参考[dtm-labs/dtm-examples](https://github.com/dtm-labs/dtm-examples)
Add wechat friend with id yedf2008, or scan the OR code. Fill in dtm as verification.
## 联系我们
### 公众号
dtm官方公众号:分布式事务,大量干货分享,以及dtm的最新消息
### 交流群
请加 yedf2008 好友或者扫码加好友,验证回复 dtm 按照指引进群
![yedf2008](http://service.ivydad.com/cover/dubbingb6b5e2c0-2d2a-cd59-f7c5-c6b90aceb6f1.jpeg)
### Give a star! ⭐
### github
作者github: [https://github.com/yedf2](https://github.com/yedf2)
If you think this project is good, or helpful to you, please give a star!
欢迎使用[dtm](https://github.com/dtm-labs/dtm),或者通过dtm学习实践分布式事务相关知识,欢迎star支持我们
### Who is using
<div style='vertical-align: middle'>
<img alt='Tencent' height='80' src='https://dtm.pub/assets/tencent.4b87bfd8.jpeg' /img>
<img alt='Ivydad' height='80' src='https://www.ivydad.com/_nuxt/img/header-logo.5b3eb96.png'>
<img alt='Eglass' height='80' src='https://img.epeijing.cn/official-website/assets/logo.png'>
<img alt='Jiou' height='80' src='http://www.siqitech.com.cn/img/logo.3f6c2914.png'>
<img alt='GoldenData' height='80' src='https://pic1.zhimg.com/80/v2-dc1d0cef5f7b72be345fc34d768e69e3_1440w.png'>
</div>

82
app/main.go

@ -1,82 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package main
import (
"fmt"
"os"
"strings"
_ "go.uber.org/automaxprocs"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmsvr"
"github.com/yedf/dtm/dtmsvr/storage/registry"
"github.com/yedf/dtm/examples"
)
var Version, Commit, Date string
var usage = `dtm is a lightweight distributed transaction manager.
usage:
dtm [command]
Available commands:
version print dtm version
dtmsvr run dtm as a server
dev create all needed table and run dtm as a server
bench start bench server
quick_start run quick start example (dtm will create needed table)
qs same as quick_start
`
func main() {
if len(os.Args) == 1 {
fmt.Println(usage)
for name := range examples.Samples {
fmt.Printf("%4s%-18srun a sample includes %s\n", "", name, strings.ReplaceAll(name, "_", " "))
}
return
}
if os.Args[1] == "version" {
fmt.Printf("version: %s commit: %s built at: %s\n", Version, Commit, Date)
return
}
dtmimp.Logf("starting dtm....")
common.MustLoadConfig()
if common.Config.ExamplesDB.Driver != "" {
dtmcli.SetCurrentDBType(common.Config.ExamplesDB.Driver)
}
if os.Args[1] != "dtmsvr" { // 实际线上运行,只启动dtmsvr,不准备table相关的数据
registry.WaitStoreUp()
dtmsvr.PopulateDB(true)
examples.PopulateDB(true)
}
dtmsvr.StartSvr() // 启动dtmsvr的api服务
go dtmsvr.CronExpiredTrans(-1) // 启动dtmsvr的定时过期查询
switch os.Args[1] {
case "quick_start", "qs":
// quick_start 比较独立,单独作为一个例子运行,方便新人上手
examples.QsStartSvr()
examples.QsFireRequest()
case "dev", "dtmsvr":
default:
// 下面是各类的例子
examples.GrpcStartup()
examples.BaseAppStartup()
sample := examples.Samples[os.Args[1]]
dtmimp.LogIfFatalf(sample == nil, "no sample name for %s", os.Args[1])
sample.Action()
}
select {}
}

28
bench/Makefile

@ -0,0 +1,28 @@
# All targets.
default: bench
bench: /usr/local/bin/go /etc/redis/redis.conf /usr/local/bin/docker-compose main.go
rm -f ../conf.sample.yml
go build -o bench
go: /usr/local/bin/go
redis: /etc/redis/redis.conf
mysql: /usr/local/bin/docker-compose
/usr/local/bin/go:
wget https://golang.org/dl/go1.17.1.linux-amd64.tar.gz
rm -rf /usr/local/go && tar -C /usr/local -xzf go1.17.1.linux-amd64.tar.gz && cp -f /usr/local/go/bin/go /usr/local/bin/go && rm go1.*
/etc/redis/redis.conf:
apt update
apt install -y redis redis-tools
/usr/local/bin/docker-compose:
apt update
apt install -y sysbench apache2-utils mysql-client-core-8.0
curl -fsSL https://get.docker.com | sh
curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
cd .. && docker-compose -f helper/compose.mysql.yml up -d && cd bench

63
bench/main.go

@ -4,38 +4,51 @@ import (
"fmt"
"os"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmsvr"
"github.com/yedf/dtm/dtmsvr/storage/registry"
"github.com/yedf/dtm/examples"
"github.com/dtm-labs/dtm/bench/svr"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmsvr"
"github.com/dtm-labs/dtm/dtmsvr/config"
"github.com/dtm-labs/dtm/dtmsvr/storage/registry"
"github.com/dtm-labs/dtm/test/busi"
)
var hint = `To start the bench server, you need to specify the parameters:
Available commands:
http start bench server
var usage = `bench is a bench test server for dtmf
usage:
redis prepare for redis bench test
db prepare for mysql|postgres bench test
boltdb prepare for boltdb bench test
`
func hintAndExit() {
fmt.Print(usage)
os.Exit(0)
}
var conf = &config.Config
func main() {
if len(os.Args) <= 1 {
fmt.Printf(hint)
return
hintAndExit()
}
logger.Infof("starting bench server")
config.MustLoadConfig("")
logger.InitLog(conf.Log.Level)
if busi.BusiConf.Driver != "" {
dtmcli.SetCurrentDBType(busi.BusiConf.Driver)
svr.PrepareBenchDB()
}
dtmimp.Logf("starting dtm....")
if os.Args[1] == "http" {
fmt.Println("start bench server")
common.MustLoadConfig()
dtmcli.SetCurrentDBType(common.Config.ExamplesDB.Driver)
registry.WaitStoreUp()
dtmsvr.PopulateDB(true)
examples.PopulateDB(true)
dtmsvr.StartSvr() // 启动dtmsvr的api服务
go dtmsvr.CronExpiredTrans(-1) // 启动dtmsvr的定时过期查询
StartSvr()
select {}
registry.WaitStoreUp()
dtmsvr.PopulateDB(false)
if os.Args[1] == "db" {
busi.PopulateDB(false)
} else if os.Args[1] == "redis" || os.Args[1] == "boltdb" {
} else {
fmt.Printf(hint)
hintAndExit()
}
dtmsvr.StartSvr() // 启动dtmsvr的api服务
go dtmsvr.CronExpiredTrans(-1) // 启动dtmsvr的定时过期查询
svr.StartSvr() // 启动bench服务
select {}
}

10
bench/prepare.sh

@ -0,0 +1,10 @@
# !/bin/bash
apt update
apt install -y git
git clone https://github.com/dtm-labs/dtm.git && cd dtm && git checkout alpha && cd bench && make
echo 'all prepared. you shoud run following commands to test in different terminal'
echo
echo 'cd dtm && go run bench/main.go redis|boltdb|db'
echo 'cd dtm && bench/run-redis|boltdb|mysql.sh'

8
bench/run-mysql.sh

@ -1,8 +0,0 @@
# !/bin/bash
cd /usr/share/sysbench/
echo 'create database sbtest;' > mysql -h 127.0.0.1 -uroot
sysbench oltp_write_only.lua --time=60 --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=root --mysql-password= --mysql-db=sbtest --table-size=1000000 --tables=10 --threads=10 --events=999999999 --report-interval=10 prepare
sysbench oltp_write_only.lua --time=60 --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=root --mysql-password= --mysql-db=sbtest --table-size=1000000 --tables=10 --threads=10 --events=999999999 --report-interval=10 run

5
bench/run-services.sh

@ -1,5 +0,0 @@
# !/bin/bash
# start all services
docker-compose -f helper/compose.mysql.yml up -d
go run app/main.go bench > /dev/nul

6
bench/setup-redis6.sh

@ -0,0 +1,6 @@
# !/bin/bash
apt update
apt install -y software-properties-common
add-apt-repository -y ppa:redislabs/redis
apt install -y redis redis-tools

5
bench/setup.sh

@ -1,8 +1,9 @@
# !/bin/bash
# install all commands needed
apt update
apt install -y git sysbench apache2-utils mysql-client-core-8.0
apt install -y sysbench apache2-utils mysql-client-core-8.0 redis redis-tools
# install docker and docker-compose
curl -fsSL https://get.docker.com -o get-docker.sh
@ -12,4 +13,4 @@ chmod +x /usr/local/bin/docker-compose
# install go
wget https://golang.org/dl/go1.17.1.linux-amd64.tar.gz
rm -rf /usr/local/go && tar -C /usr/local -xzf go1.17.1.linux-amd64.tar.gz && cp -f /usr/local/bin/go /usr/local/go/bin/go
rm -rf /usr/local/go && tar -C /usr/local -xzf go1.17.1.linux-amd64.tar.gz && cp -f /usr/local/go/bin/go /usr/local/bin/go

118
bench/http.go → bench/svr/http.go

@ -4,53 +4,56 @@
* license that can be found in the LICENSE file.
*/
package main
package svr
import (
"database/sql"
"fmt"
"os"
"strings"
"sync/atomic"
"time"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmsvr"
"github.com/dtm-labs/dtm/dtmutil"
"github.com/dtm-labs/dtm/test/busi"
"github.com/gin-gonic/gin"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmsvr"
"github.com/yedf/dtm/examples"
"github.com/lithammer/shortuuid"
)
// launch command:go run app/main.go qs
// service address of the transcation
const benchAPI = "/api/busi_bench"
const benchPort = 8083
const total = 200000
var benchBusi = fmt.Sprintf("http://localhost:%d%s", benchPort, benchAPI)
var benchPort = dtmimp.If(os.Getenv("BENCH_PORT") == "", "8083", os.Getenv("BENCH_PORT")).(string)
var benchBusi = fmt.Sprintf("http://localhost:%s%s", benchPort, benchAPI)
func sdbGet() *sql.DB {
db, err := dtmimp.PooledDB(common.Config.Store.GetDBConf())
dtmimp.FatalIfError(err)
func pdbGet() *sql.DB {
db, err := dtmimp.PooledDB(busi.BusiConf)
logger.FatalIfError(err)
return db
}
func txGet() *sql.Tx {
db := sdbGet()
db := pdbGet()
tx, err := db.Begin()
dtmimp.FatalIfError(err)
logger.FatalIfError(err)
return tx
}
func reloadData() {
time.Sleep(dtmsvr.UpdateBranchAsyncInterval * 2)
began := time.Now()
db := sdbGet()
db := pdbGet()
tables := []string{"dtm_busi.user_account", "dtm_busi.user_account_log", "dtm.trans_global", "dtm.trans_branch_op", "dtm_barrier.barrier"}
for _, t := range tables {
_, err := dtmimp.DBExec(db, fmt.Sprintf("truncate %s", t))
dtmimp.FatalIfError(err)
logger.FatalIfError(err)
}
s := "insert ignore into dtm_busi.user_account(user_id, balance) values "
ss := []string{}
@ -58,23 +61,19 @@ func reloadData() {
ss = append(ss, fmt.Sprintf("(%d, 1000000)", i))
}
_, err := dtmimp.DBExec(db, s+strings.Join(ss, ","))
dtmimp.FatalIfError(err)
dtmimp.Logf("%d users inserted. used: %dms", total, time.Since(began).Milliseconds())
logger.FatalIfError(err)
logger.Debugf("%d users inserted. used: %dms", total, time.Since(began).Milliseconds())
}
var uidCounter int32 = 0
var mode string = ""
var sqls int = 1
var uidCounter int32
var mode string
var sqls = 1
// StartSvr 1
func StartSvr() {
app := common.GetGinApp()
benchAddRoute(app)
dtmimp.Logf("bench listening at %d", benchPort)
go app.Run(fmt.Sprintf(":%d", benchPort))
db := sdbGet()
// PrepareBenchDB prepares db data for bench
func PrepareBenchDB() {
db := pdbGet()
_, err := dtmimp.DBExec(db, "drop table if exists dtm_busi.user_account_log")
dtmimp.FatalIfError(err)
logger.FatalIfError(err)
_, err = dtmimp.DBExec(db, `create table if not exists dtm_busi.user_account_log (
id INT(11) AUTO_INCREMENT PRIMARY KEY,
user_id INT(11) NOT NULL,
@ -89,70 +88,82 @@ func StartSvr() {
key(create_time)
)
`)
dtmimp.FatalIfError(err)
logger.FatalIfError(err)
}
func qsAdjustBalance(uid int, amount int, c *gin.Context) (interface{}, error) {
// StartSvr 1
func StartSvr() {
app := dtmutil.GetGinApp()
benchAddRoute(app)
logger.Debugf("bench listening at %d", benchPort)
go func() {
_ = app.Run(fmt.Sprintf(":%s", benchPort))
}()
}
func qsAdjustBalance(uid int, amount int, c *gin.Context) error { // nolint: unparam
if strings.Contains(mode, "empty") || sqls == 0 {
return dtmcli.MapSuccess, nil
return nil
}
tb := dtmimp.TransBaseFromQuery(c.Request.URL.Query())
f := func(tx *sql.Tx) error {
for i := 0; i < sqls; i++ {
_, err := dtmimp.DBExec(tx, "insert into dtm_busi.user_account_log(user_id, delta, gid, branch_id, op, reason) values(?,?,?,?,?,?)",
uid, amount, tb.Gid, c.Query("branch_id"), tb.TransType, fmt.Sprintf("inserted by dtm transaction %s %s", tb.Gid, c.Query("branch_id")))
dtmimp.FatalIfError(err)
logger.FatalIfError(err)
_, err = dtmimp.DBExec(tx, "update dtm_busi.user_account set balance = balance + ?, update_time = now() where user_id = ?", amount, uid)
dtmimp.FatalIfError(err)
logger.FatalIfError(err)
}
return nil
}
if strings.Contains(mode, "barrier") {
barrier, err := dtmcli.BarrierFromQuery(c.Request.URL.Query())
dtmimp.FatalIfError(err)
barrier.Call(txGet(), f)
logger.FatalIfError(err)
err = barrier.Call(txGet(), f)
logger.FatalIfError(err)
} else {
tx := txGet()
f(tx)
err := tx.Commit()
dtmimp.FatalIfError(err)
err := f(tx)
logger.FatalIfError(err)
err = tx.Commit()
logger.FatalIfError(err)
}
return dtmcli.MapSuccess, nil
return nil
}
func benchAddRoute(app *gin.Engine) {
app.POST(benchAPI+"/TransIn", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
app.POST(benchAPI+"/TransIn", dtmutil.WrapHandler2(func(c *gin.Context) interface{} {
return qsAdjustBalance(dtmimp.MustAtoi(c.Query("uid")), 1, c)
}))
app.POST(benchAPI+"/TransInCompensate", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
app.POST(benchAPI+"/TransInCompensate", dtmutil.WrapHandler2(func(c *gin.Context) interface{} {
return qsAdjustBalance(dtmimp.MustAtoi(c.Query("uid")), -1, c)
}))
app.POST(benchAPI+"/TransOut", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
app.POST(benchAPI+"/TransOut", dtmutil.WrapHandler2(func(c *gin.Context) interface{} {
return qsAdjustBalance(dtmimp.MustAtoi(c.Query("uid")), -1, c)
}))
app.POST(benchAPI+"/TransOutCompensate", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
app.POST(benchAPI+"/TransOutCompensate", dtmutil.WrapHandler2(func(c *gin.Context) interface{} {
return qsAdjustBalance(dtmimp.MustAtoi(c.Query("uid")), 30, c)
}))
app.Any(benchAPI+"/reloadData", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
app.Any(benchAPI+"/reloadData", dtmutil.WrapHandler2(func(c *gin.Context) interface{} {
reloadData()
mode = c.Query("m")
s := c.Query("sqls")
if s != "" {
sqls = dtmimp.MustAtoi(s)
}
return nil, nil
return nil
}))
app.Any(benchAPI+"/bench", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
app.Any(benchAPI+"/bench", dtmutil.WrapHandler2(func(c *gin.Context) interface{} {
uid := (atomic.AddInt32(&uidCounter, 1)-1)%total + 1
suid := fmt.Sprintf("%d", uid)
suid2 := fmt.Sprintf("%d", total+1-uid)
req := gin.H{}
params := fmt.Sprintf("?uid=%s", suid)
params2 := fmt.Sprintf("?uid=%s", suid2)
dtmimp.Logf("mode: %s contains dtm: %t", mode, strings.Contains(mode, "dtm"))
logger.Debugf("mode: %s contains dtm: %t", mode, strings.Contains(mode, "dtm"))
if strings.Contains(mode, "dtm") {
saga := dtmcli.NewSaga(examples.DtmHttpServer, fmt.Sprintf("bench-%d", uid)).
saga := dtmcli.NewSaga(dtmutil.DefaultHTTPServer, fmt.Sprintf("bench-%d", uid)).
Add(benchBusi+"/TransOut"+params, benchBusi+"/TransOutCompensate"+params, req).
Add(benchBusi+"/TransIn"+params2, benchBusi+"/TransInCompensate"+params2, req)
saga.WaitResult = true
@ -164,6 +175,15 @@ func benchAddRoute(app *gin.Engine) {
_, err = dtmimp.RestyClient.R().SetBody(gin.H{}).SetQueryParam("uid", suid).Post(benchBusi + "/TransIn")
dtmimp.E2P(err)
}
return nil, nil
return nil
}))
app.Any(benchAPI+"/benchEmptyUrl", dtmutil.WrapHandler2(func(c *gin.Context) interface{} {
gid := shortuuid.New()
req := gin.H{}
saga := dtmcli.NewSaga(dtmutil.DefaultHTTPServer, gid).
Add("", "", req).
Add("", "", req)
saga.WaitResult = true
return saga.Submit()
}))
}

5
bench/test-boltdb.sh

@ -0,0 +1,5 @@
# !/bin/bash
set -x
ab -n 50000 -c 10 "http://127.0.0.1:8083/api/busi_bench/benchEmptyUrl"

13
bench/run-dtm.sh → bench/test-mysql.sh

@ -1,7 +1,14 @@
# !/bin/bash
# go run ../app/main.go
set -x
cd /usr/share/sysbench/
echo 'create database sbtest;' > mysql -h 127.0.0.1 -uroot
sysbench oltp_write_only.lua --time=60 --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=root --mysql-password= --mysql-db=sbtest --table-size=1000000 --tables=10 --threads=10 --events=999999999 --report-interval=10 prepare
sysbench oltp_write_only.lua --time=60 --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=root --mysql-password= --mysql-db=sbtest --table-size=1000000 --tables=10 --threads=10 --events=999999999 --report-interval=10 run
export TIME=10
export CONCURRENT=20
curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=dtm_tx&sqls=0" && ab -t $TIME -c $CONCURRENT "http://127.0.0.1:8083/api/busi_bench/bench"
@ -13,7 +20,3 @@ curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=dtm_barrier&sqls=1" && a
curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=raw_tx&sqls=1" && ab -t $TIME -c $CONCURRENT "http://127.0.0.1:8083/api/busi_bench/bench"
curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=raw_empty" && ab -t $TIME -c $CONCURRENT "http://127.0.0.1:8083/api/busi_bench/bench"
# curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=raw_empty" && curl "http://127.0.0.1:8083/api/busi_bench/bench"
# curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=raw_tx" && curl "http://127.0.0.1:8083/api/busi_bench/bench"
# curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=dtm_tx" && curl "http://127.0.0.1:8083/api/busi_bench/bench"
# curl "http://127.0.0.1:8083/api/busi_bench/reloadData?m=dtm_barrier" && curl "http://127.0.0.1:8083/api/busi_bench/bench"

27
bench/test-redis.sh

@ -0,0 +1,27 @@
# !/bin/bash
set -x
export LOG_LEVEL=warn
export STORE_DRIVER=redis
export STORE_HOST=localhost
export STORE_PORT=6379
cd .. && bench/bench redis &
echo 'sleeping 3s for dtm bench to run up.' && sleep 3
ab -n 1000000 -c 10 "http://127.0.0.1:8083/api/busi_bench/benchEmptyUrl"
pkill bench
redis-benchmark -n 300000 SET 'abcdefg' 'ddddddd'
redis-benchmark -n 300000 EVAL "redis.call('SET', 'abcdedf', 'ddddddd')" 0
redis-benchmark -n 300000 EVAL "redis.call('SET', KEYS[1], ARGV[1])" 1 'aaaaaaaaa' 'bbbbbbbbbb'
redis-benchmark -n 3000000 -P 50 SET 'abcdefg' 'ddddddd'
redis-benchmark -n 300000 EVAL "for k=1, 10 do; redis.call('SET', KEYS[1], ARGV[1]);end" 1 'aaaaaaaaa' 'bbbbbbbbbb'
redis-benchmark -n 300000 -P 50 EVAL "redis.call('SET', KEYS[1], ARGV[1])" 1 'aaaaaaaaa' 'bbbbbbbbbb'
redis-benchmark -n 300000 EVAL "for k=1,10 do;local c = cjson.decode(ARGV[1]);end" 1 'aaaaaaaaa' '{"aaaaa":"bbbbb","b":1,"t":"2012-01-01 14:00:00"}'

120
common/config.go

@ -1,120 +0,0 @@
package common
import (
"encoding/json"
"errors"
"io/ioutil"
"path/filepath"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"gopkg.in/yaml.v2"
)
const (
DtmMetricsPort = 8889
Mysql = "mysql"
Redis = "redis"
BoltDb = "boltdb"
)
// MicroService config type for micro service
type MicroService struct {
Driver string `yaml:"Driver" default:"default"`
Target string `yaml:"Target"`
EndPoint string `yaml:"EndPoint"`
}
type Store struct {
Driver string `yaml:"Driver" default:"boltdb"`
Host string `yaml:"Host"`
Port int64 `yaml:"Port"`
User string `yaml:"User"`
Password string `yaml:"Password"`
MaxOpenConns int64 `yaml:"MaxOpenConns" default:"500"`
MaxIdleConns int64 `yaml:"MaxIdleConns" default:"500"`
ConnMaxLifeTime int64 `yaml:"ConnMaxLifeTime" default:"5"`
DataExpire int64 `yaml:"DataExpire" default:"604800"` // Trans data will expire in 7 days. only for redis/boltdb.
RedisPrefix string `yaml:"RedisPrefix" default:"{a}"` // Redis storage prefix. store data to only one slot in cluster
}
func (s *Store) IsDB() bool {
return s.Driver == dtmcli.DBTypeMysql || s.Driver == dtmcli.DBTypePostgres
}
func (s *Store) GetDBConf() dtmcli.DBConf {
return dtmcli.DBConf{
Driver: s.Driver,
Host: s.Host,
Port: s.Port,
User: s.User,
Passwrod: s.Password,
}
}
type configType struct {
Store Store `yaml:"Store"`
TransCronInterval int64 `yaml:"TransCronInterval" default:"3"`
TimeoutToFail int64 `yaml:"TimeoutToFail" default:"35"`
RetryInterval int64 `yaml:"RetryInterval" default:"10"`
HttpPort int64 `yaml:"HttpPort" default:"36789"`
GrpcPort int64 `yaml:"GrpcPort" default:"36790"`
MicroService MicroService `yaml:"MicroService"`
UpdateBranchSync int64 `yaml:"UpdateBranchSync"`
ExamplesDB dtmcli.DBConf `yaml:"ExamplesDB"`
}
// Config 配置
var Config = configType{}
func MustLoadConfig() {
loadFromEnv("", &Config)
cont := []byte{}
for d := MustGetwd(); d != "" && d != "/"; d = filepath.Dir(d) {
cont1, err := ioutil.ReadFile(d + "/conf.yml")
if err != nil {
cont1, err = ioutil.ReadFile(d + "/conf.sample.yml")
}
if cont1 != nil {
cont = cont1
break
}
}
if len(cont) != 0 {
err := yaml.UnmarshalStrict(cont, &Config)
dtmimp.FatalIfError(err)
}
scont, err := json.MarshalIndent(&Config, "", " ")
dtmimp.FatalIfError(err)
dtmimp.Logf("config is: \n%s", scont)
err = checkConfig()
dtmimp.LogIfFatalf(err != nil, `config error: '%v'.
check you env, and conf.yml/conf.sample.yml in current and parent path: %s.
please visit http://d.dtm.pub to see the config document.
loaded config is:
%v`, err, MustGetwd(), Config)
}
func checkConfig() error {
if Config.RetryInterval < 10 {
return errors.New("RetryInterval should not be less than 10")
}
if Config.TimeoutToFail < Config.RetryInterval {
return errors.New("TimeoutToFail should not be less than RetryInterval")
}
if Config.Store.Driver == BoltDb {
return nil
}
if Config.Store.Driver == Mysql {
if Config.Store.Host == "" {
return errors.New("Db host not valid ")
}
if Config.Store.Port == 0 {
return errors.New("Db port not valid ")
}
if Config.Store.User == "" {
return errors.New("Db user not valid ")
}
}
return nil
}

49
common/config_test.go

@ -1,49 +0,0 @@
package common
import (
"errors"
"os"
"testing"
"github.com/go-playground/assert/v2"
)
func TestLoadFromEnv(t *testing.T) {
assert.Equal(t, "MICRO_SERVICE_DRIVER", toUnderscoreUpper("MicroService_Driver"))
ms := MicroService{}
os.Setenv("T_DRIVER", "d1")
loadFromEnv("T", &ms)
assert.Equal(t, "d1", ms.Driver)
}
func TestCheckConfig(t *testing.T) {
config := &Config
retryIntervalErr := checkConfig()
retryIntervalExpect := errors.New("RetryInterval should not be less than 10")
assert.Equal(t, retryIntervalErr, retryIntervalExpect)
config.RetryInterval = 10
timeoutToFailErr := checkConfig()
timeoutToFailExpect := errors.New("TimeoutToFail should not be less than RetryInterval")
assert.Equal(t, timeoutToFailErr, timeoutToFailExpect)
config.TimeoutToFail = 20
driverErr := checkConfig()
assert.Equal(t, driverErr, nil)
config.Store = Store{Driver: Mysql}
hostErr := checkConfig()
hostExpect := errors.New("Db host not valid ")
assert.Equal(t, hostErr, hostExpect)
config.Store = Store{Driver: Mysql, Host: "127.0.0.1"}
portErr := checkConfig()
portExpect := errors.New("Db port not valid ")
assert.Equal(t, portErr, portExpect)
config.Store = Store{Driver: Mysql, Host: "127.0.0.1", Port: 8686}
userErr := checkConfig()
userExpect := errors.New("Db user not valid ")
assert.Equal(t, userErr, userExpect)
}

30
common/types.go

@ -1,30 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package common
import (
"fmt"
"sync"
"github.com/go-redis/redis/v8"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
var rdb *redis.Client
var once sync.Once
func RedisGet() *redis.Client {
once.Do(func() {
dtmimp.Logf("connecting to redis: %v", Config.Store)
rdb = redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%d", Config.Store.Host, Config.Store.Port),
Username: Config.Store.User,
Password: Config.Store.Password,
})
})
return rdb
}

66
common/types_test.go

@ -1,66 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package common
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
func TestGeneralDB(t *testing.T) {
MustLoadConfig()
if Config.Store.IsDB() {
testSql(t)
testDbAlone(t)
}
}
func testSql(t *testing.T) {
db := DbGet(Config.Store.GetDBConf())
err := func() (rerr error) {
defer dtmimp.P2E(&rerr)
db.Must().Exec("select a")
return nil
}()
assert.NotEqual(t, nil, err)
}
func testDbAlone(t *testing.T) {
db, err := dtmimp.StandaloneDB(Config.Store.GetDBConf())
assert.Nil(t, err)
_, err = dtmimp.DBExec(db, "select 1")
assert.Equal(t, nil, err)
_, err = dtmimp.DBExec(db, "")
assert.Equal(t, nil, err)
db.Close()
_, err = dtmimp.DBExec(db, "select 1")
assert.NotEqual(t, nil, err)
}
func TestConfig(t *testing.T) {
testConfigStringField(&Config.Store.Driver, "", t)
testConfigStringField(&Config.Store.User, "", t)
testConfigIntField(&Config.RetryInterval, 9, t)
testConfigIntField(&Config.TimeoutToFail, 9, t)
}
func testConfigStringField(fd *string, val string, t *testing.T) {
old := *fd
*fd = val
str := checkConfig()
assert.NotEqual(t, "", str)
*fd = old
}
func testConfigIntField(fd *int64, val int64, t *testing.T) {
old := *fd
*fd = val
str := checkConfig()
assert.NotEqual(t, "", str)
*fd = old
}

121
common/utils.go

@ -1,121 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package common
import (
"bytes"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/go-resty/resty/v2"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
// GetGinApp init and return gin
func GetGinApp() *gin.Engine {
gin.SetMode(gin.ReleaseMode)
app := gin.Default()
app.Use(func(c *gin.Context) {
body := ""
if c.Request.Body != nil {
rb, err := c.GetRawData()
dtmimp.E2P(err)
if len(rb) > 0 {
body = string(rb)
c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(rb))
}
}
began := time.Now()
dtmimp.Logf("begin %s %s query: %s body: %s", c.Request.Method, c.FullPath(), c.Request.URL.RawQuery, body)
c.Next()
dtmimp.Logf("used %d ms %s %s query: %s body: %s", time.Since(began).Milliseconds(), c.Request.Method, c.FullPath(), c.Request.URL.RawQuery, body)
})
app.Any("/api/ping", func(c *gin.Context) { c.JSON(200, map[string]interface{}{"msg": "pong"}) })
return app
}
// WrapHandler name is clear
func WrapHandler(fn func(*gin.Context) (interface{}, error)) gin.HandlerFunc {
return func(c *gin.Context) {
r, err := func() (r interface{}, rerr error) {
defer dtmimp.P2E(&rerr)
return fn(c)
}()
var b = []byte{}
if resp, ok := r.(*resty.Response); ok { // 如果是response,则取出body直接处理
b = resp.Body()
} else if err == nil {
b, err = json.Marshal(r)
}
if err != nil {
dtmimp.Logf("status: 500, code: 500 message: %s", err.Error())
c.JSON(500, map[string]interface{}{"code": 500, "message": err.Error()})
} else {
dtmimp.Logf("status: 200, content: %s", string(b))
c.Status(200)
c.Writer.Header().Add("Content-Type", "application/json")
_, err = c.Writer.Write(b)
dtmimp.E2P(err)
}
}
}
// MustGetwd must version of os.Getwd
func MustGetwd() string {
wd, err := os.Getwd()
dtmimp.E2P(err)
return wd
}
// GetSqlDir 获取调用该函数的caller源代码的目录,主要用于测试时,查找相关文件
func GetSqlDir() string {
wd := MustGetwd()
if filepath.Base(wd) == "test" {
wd = filepath.Dir(wd)
}
return wd + "/sqls"
}
func RecoverPanic(err *error) {
if x := recover(); x != nil {
e := dtmimp.AsError(x)
if err != nil {
*err = e
}
}
}
func GetNextTime(second int64) *time.Time {
next := time.Now().Add(time.Duration(second) * time.Second)
return &next
}
// RunSQLScript 1
func RunSQLScript(conf dtmcli.DBConf, script string, skipDrop bool) {
con, err := dtmimp.StandaloneDB(conf)
dtmimp.FatalIfError(err)
defer func() { con.Close() }()
content, err := ioutil.ReadFile(script)
dtmimp.FatalIfError(err)
sqls := strings.Split(string(content), ";")
for _, sql := range sqls {
s := strings.TrimSpace(sql)
if s == "" || (skipDrop && strings.Contains(s, "drop")) {
continue
}
_, err = dtmimp.DBExec(con, s)
dtmimp.FatalIfError(err)
}
}

48
conf.sample.yml

@ -1,4 +1,11 @@
Store: # specify which engine to store trans status
#####################################################################
### dtm can be run without any config.
### all config in this file is optional. the default value is as specified in each line
### all configs can be specified from env. for example:
### MicroService.EndPoint => MICRO_SERVICE_END_POINT
#####################################################################
# Store: # specify which engine to store trans status
# Driver: 'boltdb' # default store engine
# Driver: 'redis'
@ -7,11 +14,11 @@ Store: # specify which engine to store trans status
# Password: ''
# Port: 6379
Driver: 'mysql'
Host: 'localhost'
User: 'root'
Password: ''
Port: 3306
# Driver: 'mysql'
# Host: 'localhost'
# User: 'root'
# Password: ''
# Port: 3306
# Driver: 'postgres'
# Host: 'localhost'
@ -19,10 +26,12 @@ Store: # specify which engine to store trans status
# Password: 'mysecretpassword'
# Port: '5432'
### following connection config is for only Driver postgres/mysql
### following config is for only Driver postgres/mysql
# MaxOpenConns: 500
# MaxIdleConns: 500
# ConnMaxLifeTime 5 # default value is 5 (minutes)
# TransGlobalTable: 'dtm.trans_global'
# TransBranchOpTable: 'dtm.trans_branch_op'
### flollowing config is only for some Driver
# DataExpire: 604800 # Trans data will expire in 7 days. only for redis/boltdb.
@ -33,16 +42,23 @@ Store: # specify which engine to store trans status
# Target: 'etcd://localhost:2379/dtmservice' # register dtm server to this url
# EndPoint: 'localhost:36790'
# the unit of following configurations is second
### the unit of following configurations is second
# TransCronInterval: 3 # the interval to poll unfinished global transaction for every dtm process
# TimeoutToFail: 35 # timeout for XA, TCC to fail. saga's timeout default to infinite, which can be overwritten in saga options
# RetryInterval: 10 # the subtrans branch will be retried after this interval
# RequestTimeout: 3 # the timeout of HTTP/gRPC request in dtm
# Log:
# Level: 'info' # default: info. can be debug|info|warn|error
# Output: 'console' # default: console. can be console|file
# FileName: '/tmp/dtm.log' # default: /tmp/dtm.log.
# FileMaxSize: 10 # default: 10, unit: MB.
# FileMaxBackups: 5 # default: 5.
# FileMaxAge: 30 # default: 30, unit: days.
# FileCompress: 0 # default: 0. can by 0|1, means false|true
# HttpPort: 36789
# GrpcPort: 36790
### dtm can run examples, and examples will use following config to connect db
ExamplesDB:
Driver: 'mysql'
Host: 'localhost'
User: 'root'
Password: ''
Port: 3306
### advanced options
# UpdateBranchAsyncGoroutineNum: 1 # num of async goroutine to update branch status

81
dtmcli/barrier.go

@ -11,7 +11,9 @@ import (
"fmt"
"net/url"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/go-redis/redis/v8"
)
// BarrierBusiFunc type for busi func
@ -44,7 +46,7 @@ func BarrierFrom(transType, gid, branchID, op string) (*BranchBarrier, error) {
Op: op,
}
if ti.TransType == "" || ti.Gid == "" || ti.BranchID == "" || ti.Op == "" {
return nil, fmt.Errorf("invlid trans info: %v", ti)
return nil, fmt.Errorf("invalid trans info: %v", ti)
}
return ti, nil
}
@ -53,7 +55,7 @@ func insertBarrier(tx DB, transType string, gid string, branchID string, op stri
if op == "" {
return 0, nil
}
sql := dtmimp.GetDBSpecial().GetInsertIgnoreTemplate("dtm_barrier.barrier(trans_type, gid, branch_id, op, barrier_id, reason) values(?,?,?,?,?,?)", "uniq_barrier")
sql := dtmimp.GetDBSpecial().GetInsertIgnoreTemplate(dtmimp.BarrierTableName+"(trans_type, gid, branch_id, op, barrier_id, reason) values(?,?,?,?,?,?)", "uniq_barrier")
return dtmimp.DBExec(tx, sql, transType, gid, branchID, op, barrierID, reason)
}
@ -66,23 +68,23 @@ func (bb *BranchBarrier) Call(tx *sql.Tx, busiCall BarrierBusiFunc) (rerr error)
defer func() {
// Logf("barrier call error is %v", rerr)
if x := recover(); x != nil {
tx.Rollback()
_ = tx.Rollback()
panic(x)
} else if rerr != nil {
tx.Rollback()
_ = tx.Rollback()
} else {
tx.Commit()
rerr = tx.Commit()
}
}()
ti := bb
originType := map[string]string{
originOp := map[string]string{
BranchCancel: BranchTry,
BranchCompensate: BranchAction,
}[ti.Op]
originAffected, _ := insertBarrier(tx, ti.TransType, ti.Gid, ti.BranchID, originType, bid, ti.Op)
originAffected, _ := insertBarrier(tx, ti.TransType, ti.Gid, ti.BranchID, originOp, bid, ti.Op)
currentAffected, rerr := insertBarrier(tx, ti.TransType, ti.Gid, ti.BranchID, ti.Op, bid, ti.Op)
dtmimp.Logf("originAffected: %d currentAffected: %d", originAffected, currentAffected)
logger.Debugf("originAffected: %d currentAffected: %d", originAffected, currentAffected)
if (ti.Op == BranchCancel || ti.Op == BranchCompensate) && originAffected > 0 || // 这个是空补偿
currentAffected == 0 { // 这个是重复请求或者悬挂
return
@ -94,8 +96,63 @@ func (bb *BranchBarrier) Call(tx *sql.Tx, busiCall BarrierBusiFunc) (rerr error)
// CallWithDB the same as Call, but with *sql.DB
func (bb *BranchBarrier) CallWithDB(db *sql.DB, busiCall BarrierBusiFunc) error {
tx, err := db.Begin()
if err != nil {
return err
if err == nil {
err = bb.Call(tx, busiCall)
}
return bb.Call(tx, busiCall)
return err
}
// QueryPrepared queries prepared data
func (bb *BranchBarrier) QueryPrepared(db *sql.DB) error {
_, err := insertBarrier(db, bb.TransType, bb.Gid, "00", "msg", "01", "rollback")
var reason string
if err == nil {
sql := fmt.Sprintf("select reason from %s where gid=? and branch_id=? and op=? and barrier_id=?", dtmimp.BarrierTableName)
err = db.QueryRow(sql, bb.Gid, "00", "msg", "01").Scan(&reason)
}
if reason == "rollback" {
return ErrFailure
}
return err
}
// RedisCheckAdjustAmount check the value of key is valid and >= amount. then adjust the amount
func (bb *BranchBarrier) RedisCheckAdjustAmount(rd *redis.Client, key string, amount int, barrierExpire int) error {
bkey1 := fmt.Sprintf("%s-%s-%s-%s-%02d", key, bb.Gid, bb.BranchID, bb.Op, bb.BarrierID)
originOp := map[string]string{
BranchCancel: BranchTry,
BranchCompensate: BranchAction,
}[bb.Op]
bkey2 := fmt.Sprintf("%s-%s-%s-%s-%02d", key, bb.Gid, bb.BranchID, originOp, bb.BarrierID)
v, err := rd.Eval(rd.Context(), ` -- RedisCheckAdjustAmount
local v = redis.call('GET', KEYS[1])
local e1 = redis.call('GET', KEYS[2])
if v == false or v + ARGV[1] < 0 then
return 'FAILURE'
end
if e1 ~= false then
return
end
redis.call('SET', KEYS[2], 'op', 'EX', ARGV[3])
if ARGV[2] ~= '' then
local e2 = redis.call('GET', KEYS[3])
if e2 == false then
redis.call('SET', KEYS[3], 'rollback', 'EX', ARGV[3])
return
end
end
redis.call('INCRBY', KEYS[1], ARGV[1])
`, []string{key, bkey1, bkey2}, amount, originOp, barrierExpire).Result()
logger.Debugf("lua return v: %v err: %v", v, err)
if err == redis.Nil {
err = nil
}
if err == nil && v == ResultFailure {
err = ErrFailure
}
return err
}

2
dtmcli/consts.go

@ -7,7 +7,7 @@
package dtmcli
import (
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
)
const (

2
dtmcli/dtmimp/README.md

@ -0,0 +1,2 @@
## 注意
此包带imp后缀,主要被dtm内部使用,相关接口可能会发生变更,请勿使用这里的接口

31
dtmcli/dtmimp/trans_base.go

@ -9,6 +9,7 @@ package dtmimp
import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
@ -40,9 +41,11 @@ func (g *BranchIDGen) CurrentSubBranchID() string {
// TransOptions transaction options
type TransOptions struct {
WaitResult bool `json:"wait_result,omitempty" gorm:"-"`
TimeoutToFail int64 `json:"timeout_to_fail,omitempty" gorm:"-"` // for trans type: xa, tcc
RetryInterval int64 `json:"retry_interval,omitempty" gorm:"-"` // for trans type: msg saga xa tcc
WaitResult bool `json:"wait_result,omitempty" gorm:"-"`
TimeoutToFail int64 `json:"timeout_to_fail,omitempty" gorm:"-"` // for trans type: xa, tcc
RetryInterval int64 `json:"retry_interval,omitempty" gorm:"-"` // for trans type: msg saga xa tcc
PassthroughHeaders []string `json:"passthrough_headers,omitempty" gorm:"-"`
BranchHeaders map[string]string `json:"branch_headers,omitempty" gorm:"-"`
}
// TransBase base for all trans
@ -62,18 +65,14 @@ type TransBase struct {
QueryPrepared string `json:"query_prepared,omitempty"` // used in MSG
}
// SetOptions set options
func (tb *TransBase) SetOptions(options *TransOptions) {
tb.TransOptions = *options
}
// NewTransBase new a TransBase
func NewTransBase(gid string, transType string, dtm string, branchID string) *TransBase {
return &TransBase{
Gid: gid,
TransType: transType,
BranchIDGen: BranchIDGen{BranchID: branchID},
Dtm: dtm,
Gid: gid,
TransType: transType,
BranchIDGen: BranchIDGen{BranchID: branchID},
Dtm: dtm,
TransOptions: TransOptions{PassthroughHeaders: PassthroughHeaders},
}
}
@ -89,7 +88,7 @@ func TransCallDtm(tb *TransBase, body interface{}, operation string) error {
if err != nil {
return err
}
if !strings.Contains(resp.String(), ResultSuccess) {
if resp.StatusCode() != http.StatusOK || strings.Contains(resp.String(), ResultFailure) {
return errors.New(resp.String())
}
return nil
@ -118,6 +117,10 @@ func TransRequestBranch(t *TransBase, body interface{}, branchID string, op stri
"trans_type": t.TransType,
"op": op,
}).
SetHeaders(t.BranchHeaders).
Post(url)
return resp, CheckResponse(resp, err)
if err == nil {
err = RespAsErrorCompatible(resp)
}
return resp, err
}

12
dtmcli/dtmimp/trans_xa_base.go

@ -11,7 +11,7 @@ import (
"strings"
)
// XaClientBase XaClient/XaGrpcClient base
// XaClientBase XaClient/XaGrpcClient base. shared by http and grpc
type XaClientBase struct {
Server string
Conf DBConf
@ -24,24 +24,26 @@ func (xc *XaClientBase) HandleCallback(gid string, branchID string, action strin
if err != nil {
return err
}
defer db.Close()
defer func() {
_ = db.Close()
}()
xaID := gid + "-" + branchID
_, err = DBExec(db, GetDBSpecial().GetXaSQL(action, xaID))
if err != nil &&
(strings.Contains(err.Error(), "XAER_NOTA") || strings.Contains(err.Error(), "does not exist")) { // 重复commit/rollback同一个id,报这个错误,忽略
(strings.Contains(err.Error(), "XAER_NOTA") || strings.Contains(err.Error(), "does not exist")) { // Repeat commit/rollback with the same id, report this error, ignore
err = nil
}
return err
}
// HandleLocalTrans http/grpc 处理LocalTransaction的公共方法
// HandleLocalTrans public handler of LocalTransaction via http/grpc
func (xc *XaClientBase) HandleLocalTrans(xa *TransBase, cb func(*sql.DB) error) (rerr error) {
xaBranch := xa.Gid + "-" + xa.BranchID
db, rerr := StandaloneDB(xc.Conf)
if rerr != nil {
return
}
defer func() { db.Close() }()
defer func() { _ = db.Close() }()
defer func() {
x := recover()
_, err := DBExec(db, GetDBSpecial().GetXaSQL("end", xaBranch))

3
dtmcli/dtmimp/types.go

@ -14,10 +14,11 @@ type DB interface {
QueryRow(query string, args ...interface{}) *sql.Row
}
// DBConf defines db config
type DBConf struct {
Driver string `yaml:"Driver"`
Host string `yaml:"Host"`
Port int64 `yaml:"Port"`
User string `yaml:"User"`
Passwrod string `yaml:"Password"`
Password string `yaml:"Password"`
}

1
dtmcli/dtmimp/types_test.go

@ -22,4 +22,5 @@ func TestTypes(t *testing.T) {
idGen := BranchIDGen{subBranchID: 99}
idGen.NewSubBranchID()
})
assert.Error(t, err)
}

133
dtmcli/dtmimp/utils.go

@ -11,23 +11,37 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"os"
"runtime"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
// Logf an alias of Infof
// Deprecated: use logger.Errorf
var Logf = logger.Infof
// LogRedf an alias of Errorf
// Deprecated: use logger.Errorf
var LogRedf = logger.Errorf
// FatalIfError fatal if error is not nil
// Deprecated: use logger.FatalIfError
var FatalIfError = logger.FatalIfError
// LogIfFatalf fatal if cond is true
// Deprecated: use logger.FatalfIf
var LogIfFatalf = logger.FatalfIf
// AsError wrap a panic value as an error
func AsError(x interface{}) error {
LogRedf("panic wrapped to error: '%v'", x)
logger.Errorf("panic wrapped to error: '%v'", x)
if e, ok := x.(error); ok {
return e
}
@ -120,59 +134,6 @@ func MustRemarshal(from interface{}, to interface{}) {
E2P(err)
}
var logger *zap.SugaredLogger = nil
func init() {
InitLog()
}
// InitLog is a initialization for a logger
func InitLog() {
config := zap.NewProductionConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
if os.Getenv("DTM_DEBUG") != "" {
config.Encoding = "console"
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
}
p, err := config.Build(zap.AddCallerSkip(1))
if err != nil {
log.Fatal("create logger failed: ", err)
}
logger = p.Sugar()
}
// Logf is log stdout
func Logf(fmt string, args ...interface{}) {
logger.Infof(fmt, args...)
}
// LogRedf is print error message with red color
func LogRedf(fmt string, args ...interface{}) {
logger.Errorf(fmt, args...)
}
// FatalExitFunc is a Fatal exit function ,it will be replaced when testing
var FatalExitFunc = func() { os.Exit(1) }
// LogFatalf is print error message with red color, and execute FatalExitFunc
func LogFatalf(fmt string, args ...interface{}) {
fmt += "\n" + string(debug.Stack())
LogRedf(fmt, args...)
FatalExitFunc()
}
// LogIfFatalf is print error message with red color, and execute LogFatalf, when condition is true
func LogIfFatalf(condition bool, fmt string, args ...interface{}) {
if condition {
LogFatalf(fmt, args...)
}
}
// FatalIfError is print error message with red color, and execute LogIfFatalf.
func FatalIfError(err error) {
LogIfFatalf(err != nil, "Fatal error: %v", err)
}
// GetFuncName get current call func name
func GetFuncName() string {
pc, _, _, _ := runtime.Caller(1)
@ -208,7 +169,7 @@ func PooledDB(conf DBConf) (*sql.DB, error) {
// StandaloneDB get a standalone db instance
func StandaloneDB(conf DBConf) (*sql.DB, error) {
dsn := GetDsn(conf)
Logf("opening standalone %s: %s", conf.Driver, strings.Replace(dsn, conf.Passwrod, "****", 1))
logger.Infof("opening standalone %s: %s", conf.Driver, strings.Replace(dsn, conf.Password, "****", 1))
return sql.Open(conf.Driver, dsn)
}
@ -223,9 +184,9 @@ func DBExec(db DB, sql string, values ...interface{}) (affected int64, rerr erro
used := time.Since(began) / time.Millisecond
if rerr == nil {
affected, rerr = r.RowsAffected()
Logf("used: %d ms affected: %d for %s %v", used, affected, sql, values)
logger.Debugf("used: %d ms affected: %d for %s %v", used, affected, sql, values)
} else {
LogRedf("used: %d ms exec error: %v for %s %v", used, rerr, sql, values)
logger.Errorf("used: %d ms exec error: %v for %s %v", used, rerr, sql, values)
}
return
}
@ -235,46 +196,26 @@ func GetDsn(conf DBConf) string {
host := MayReplaceLocalhost(conf.Host)
driver := conf.Driver
dsn := map[string]string{
"mysql": fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=true&loc=Local",
conf.User, conf.Passwrod, host, conf.Port, ""),
"mysql": fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=true&loc=Local&interpolateParams=true",
conf.User, conf.Password, host, conf.Port, ""),
"postgres": fmt.Sprintf("host=%s user=%s password=%s dbname='%s' port=%d sslmode=disable",
host, conf.User, conf.Passwrod, "", conf.Port),
host, conf.User, conf.Password, "", conf.Port),
}[driver]
PanicIf(dsn == "", fmt.Errorf("unknow driver: %s", driver))
return dsn
}
// CheckResponse is check response, and return corresponding error by the condition of resp when err is nil. Otherwise, return err directly.
func CheckResponse(resp *resty.Response, err error) error {
if err == nil && resp != nil {
if resp.IsError() {
return errors.New(resp.String())
} else if strings.Contains(resp.String(), ResultFailure) {
return ErrFailure
} else if strings.Contains(resp.String(), ResultOngoing) {
return ErrOngoing
}
// RespAsErrorCompatible translate a resty response to error
// compatible with version < v1.10
func RespAsErrorCompatible(resp *resty.Response) error {
code := resp.StatusCode()
str := resp.String()
if code == http.StatusTooEarly || strings.Contains(str, ResultOngoing) {
return fmt.Errorf("%s. %w", str, ErrOngoing)
} else if code == http.StatusConflict || strings.Contains(str, ResultFailure) {
return fmt.Errorf("%s. %w", str, ErrFailure)
} else if code != http.StatusOK {
return errors.New(str)
}
return err
}
// CheckResult is check result. Return err directly if err is not nil. And return corresponding error by calling CheckResponse if resp is the type of *resty.Response.
// Otherwise, return error by value of str, the string after marshal.
func CheckResult(res interface{}, err error) error {
if err != nil {
return err
}
resp, ok := res.(*resty.Response)
if ok {
return CheckResponse(resp, err)
}
if res != nil {
str := MustMarshalString(res)
if strings.Contains(str, ResultFailure) {
return ErrFailure
} else if strings.Contains(str, ResultOngoing) {
return ErrOngoing
}
}
return err
return nil
}

18
dtmcli/dtmimp/utils_test.go

@ -8,7 +8,6 @@ package dtmimp
import (
"errors"
"fmt"
"os"
"strings"
"testing"
@ -80,20 +79,3 @@ func TestSome(t *testing.T) {
s2 := MayReplaceLocalhost("http://localhost")
assert.Equal(t, "http://localhost", s2)
}
func TestFatal(t *testing.T) {
old := FatalExitFunc
defer func() {
FatalExitFunc = old
}()
FatalExitFunc = func() { panic(fmt.Errorf("fatal")) }
err := CatchP(func() {
LogIfFatalf(true, "")
})
assert.Error(t, err, fmt.Errorf("fatal"))
}
func TestInitLog(t *testing.T) {
os.Setenv("DTM_DEBUG", "1")
InitLog()
}

18
dtmcli/dtmimp/vars.go

@ -9,6 +9,7 @@ package dtmimp
import (
"errors"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/go-resty/resty/v2"
)
@ -18,8 +19,8 @@ var ErrFailure = errors.New("FAILURE")
// ErrOngoing error of ONGOING
var ErrOngoing = errors.New("ONGOING")
// XaSqlTimeoutMs milliseconds for Xa sql to timeout
var XaSqlTimeoutMs = 15000
// XaSQLTimeoutMs milliseconds for Xa sql to timeout
var XaSQLTimeoutMs = 15000
// MapSuccess HTTP result of SUCCESS
var MapSuccess = map[string]interface{}{"dtm_result": ResultSuccess}
@ -30,18 +31,21 @@ var MapFailure = map[string]interface{}{"dtm_result": ResultFailure}
// RestyClient the resty object
var RestyClient = resty.New()
// PassthroughHeaders will be passed to every sub-trans call
var PassthroughHeaders = []string{}
// BarrierTableName the table name of barrier table
var BarrierTableName = "dtm_barrier.barrier"
func init() {
// RestyClient.SetTimeout(3 * time.Second)
// RestyClient.SetRetryCount(2)
// RestyClient.SetRetryWaitTime(1 * time.Second)
RestyClient.OnBeforeRequest(func(c *resty.Client, r *resty.Request) error {
r.URL = MayReplaceLocalhost(r.URL)
Logf("requesting: %s %s %v %v", r.Method, r.URL, r.Body, r.QueryParam)
logger.Debugf("requesting: %s %s %s", r.Method, r.URL, MustMarshalString(r.Body))
return nil
})
RestyClient.OnAfterResponse(func(c *resty.Client, resp *resty.Response) error {
r := resp.Request
Logf("requested: %s %s %s", r.Method, r.URL, resp.String())
logger.Debugf("requested: %s %s %s", r.Method, r.URL, resp.String())
return nil
})
}

111
dtmcli/logger/log.go

@ -0,0 +1,111 @@
package logger
import (
"fmt"
"log"
"net/url"
"os"
"github.com/natefinch/lumberjack"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
//var logger *zap.SugaredLogger = nil
var logger Logger
func init() {
InitLog(os.Getenv("LOG_LEVEL"))
}
// Logger logger interface
type Logger interface {
Debugf(format string, args ...interface{})
Infof(format string, args ...interface{})
Warnf(format string, args ...interface{})
Errorf(format string, args ...interface{})
}
type lumberjackSink struct {
*lumberjack.Logger
}
func (lumberjackSink) Sync() error {
return nil
}
// WithLogger replaces default logger
func WithLogger(log Logger) {
logger = log
}
// InitLog is an initialization for a logger
// level can be: debug info warn error
func InitLog(level string) {
config := loadConfig(level)
p, err := config.Build(zap.AddCallerSkip(1))
FatalIfError(err)
logger = p.Sugar()
}
// InitRotateLog is an initialization for a rotated logger by lumberjack
func InitRotateLog(logLevel string, ll *lumberjack.Logger) {
config := loadConfig(logLevel)
config.OutputPaths = []string{fmt.Sprintf("lumberjack:%s", ll.Filename), "stdout"}
err := zap.RegisterSink("lumberjack", func(*url.URL) (zap.Sink, error) {
return lumberjackSink{
Logger: ll,
}, nil
})
FatalIfError(err)
p, err := config.Build(zap.AddCallerSkip(1))
FatalIfError(err)
logger = p.Sugar()
}
func loadConfig(logLevel string) zap.Config {
config := zap.NewProductionConfig()
err := config.Level.UnmarshalText([]byte(logLevel))
FatalIfError(err)
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
if os.Getenv("DTM_DEBUG") != "" {
config.Encoding = "console"
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
}
return config
}
// Debugf log to level debug
func Debugf(fmt string, args ...interface{}) {
logger.Debugf(fmt, args...)
}
// Infof log to level info
func Infof(fmt string, args ...interface{}) {
logger.Infof(fmt, args...)
}
// Warnf log to level warn
func Warnf(fmt string, args ...interface{}) {
logger.Warnf(fmt, args...)
}
// Errorf log to level error
func Errorf(fmt string, args ...interface{}) {
logger.Errorf(fmt, args...)
}
// FatalfIf log to level error
func FatalfIf(cond bool, fmt string, args ...interface{}) {
if !cond {
return
}
log.Fatalf(fmt, args...)
}
// FatalIfError if err is not nil, then log to level fatal and call os.Exit
func FatalIfError(err error) {
FatalfIf(err != nil, "fatal error: %v", err)
}

52
dtmcli/logger/logger_test.go

@ -0,0 +1,52 @@
package logger
import (
"os"
"testing"
"github.com/natefinch/lumberjack"
"go.uber.org/zap"
)
func TestInitLog(t *testing.T) {
os.Setenv("DTM_DEBUG", "1")
InitLog("debug")
Debugf("a debug msg")
Infof("a info msg")
Warnf("a warn msg")
Errorf("a error msg")
FatalfIf(false, "nothing")
FatalIfError(nil)
}
func TestWithLogger(t *testing.T) {
logger := zap.NewExample().Sugar()
WithLogger(logger)
Debugf("a debug msg")
Infof("a info msg")
Warnf("a warn msg")
Errorf("a error msg")
FatalfIf(false, "nothing")
FatalIfError(nil)
}
func TestInitRotateLog(t *testing.T) {
os.Setenv("DTM_DEBUG", "1")
ll := lumberjack.Logger{
Filename: "test.log",
MaxSize: 1,
MaxBackups: 1,
MaxAge: 1,
Compress: false,
}
InitRotateLog("debug", &ll)
Debugf("a debug msg")
Infof("a info msg")
Warnf("a warn msg")
Errorf("a error msg")
FatalfIf(false, "nothing")
FatalIfError(nil)
s := lumberjackSink{&ll}
_ = s.Sync()
_ = os.Remove("test.log")
}

26
dtmcli/msg.go

@ -6,7 +6,11 @@
package dtmcli
import "github.com/yedf/dtm/dtmcli/dtmimp"
import (
"database/sql"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
)
// Msg reliable msg type
type Msg struct {
@ -35,3 +39,23 @@ func (s *Msg) Prepare(queryPrepared string) error {
func (s *Msg) Submit() error {
return dtmimp.TransCallDtm(&s.TransBase, s, "submit")
}
// PrepareAndSubmit one method for the entire busi->prepare->submit
func (s *Msg) PrepareAndSubmit(queryPrepared string, db *sql.DB, busiCall BarrierBusiFunc) error {
bb, err := BarrierFrom(s.TransType, s.Gid, "00", "msg") // a special barrier for msg QueryPrepared
if err == nil {
err = s.Prepare(queryPrepared)
}
if err == nil {
defer func() {
if err != nil && bb.QueryPrepared(db) == ErrFailure {
_ = dtmimp.TransCallDtm(&s.TransBase, s, "abort")
}
}()
err = bb.CallWithDB(db, busiCall)
}
if err == nil {
err = s.Submit()
}
return err
}

2
dtmcli/saga.go

@ -7,7 +7,7 @@
package dtmcli
import (
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
)
// Saga struct of saga

10
dtmcli/tcc.go

@ -10,8 +10,8 @@ import (
"fmt"
"net/url"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/go-resty/resty/v2"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
// Tcc struct of tcc
@ -24,10 +24,16 @@ type TccGlobalFunc func(tcc *Tcc) (*resty.Response, error)
// TccGlobalTransaction begin a tcc global transaction
// dtm dtm server address
// gid 全局事务id
// gid global transaction ID
// tccFunc tcc事务函数,里面会定义全局事务的分支
func TccGlobalTransaction(dtm string, gid string, tccFunc TccGlobalFunc) (rerr error) {
return TccGlobalTransaction2(dtm, gid, func(t *Tcc) {}, tccFunc)
}
// TccGlobalTransaction2 new version of TccGlobalTransaction, add custom param
func TccGlobalTransaction2(dtm string, gid string, custom func(*Tcc), tccFunc TccGlobalFunc) (rerr error) {
tcc := &Tcc{TransBase: *dtmimp.NewTransBase(gid, "tcc", dtm, "")}
custom(tcc)
rerr = dtmimp.TransCallDtm(&tcc.TransBase, tcc, "prepare")
if rerr != nil {
return rerr

44
dtmcli/types.go

@ -9,7 +9,8 @@ package dtmcli
import (
"fmt"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/go-resty/resty/v2"
)
// MustGenGid generate a new gid
@ -28,8 +29,19 @@ type DB = dtmimp.DB
// TransOptions transaction option
type TransOptions = dtmimp.TransOptions
// DBConf declares db configuration
type DBConf = dtmimp.DBConf
// String2DtmError translate string to dtm error
func String2DtmError(str string) error {
return map[string]error{
ResultFailure: ErrFailure,
ResultOngoing: ErrOngoing,
ResultSuccess: nil,
"": nil,
}[str]
}
// SetCurrentDBType set currentDBType
func SetCurrentDBType(dbType string) {
dtmimp.SetCurrentDBType(dbType)
@ -40,12 +52,30 @@ func GetCurrentDBType() string {
return dtmimp.GetCurrentDBType()
}
// SetXaSqlTimeoutMs set XaSqlTimeoutMs
func SetXaSqlTimeoutMs(ms int) {
dtmimp.XaSqlTimeoutMs = ms
// SetXaSQLTimeoutMs set XaSQLTimeoutMs
func SetXaSQLTimeoutMs(ms int) {
dtmimp.XaSQLTimeoutMs = ms
}
// GetXaSQLTimeoutMs get XaSQLTimeoutMs
func GetXaSQLTimeoutMs() int {
return dtmimp.XaSQLTimeoutMs
}
// SetBarrierTableName sets barrier table name
func SetBarrierTableName(tablename string) {
dtmimp.BarrierTableName = tablename
}
// GetRestyClient get the resty.Client for http request
func GetRestyClient() *resty.Client {
return dtmimp.RestyClient
}
// GetXaSqlTimeoutMs get XaSqlTimeoutMs
func GetXaSqlTimeoutMs() int {
return dtmimp.XaSqlTimeoutMs
// SetPassthroughHeaders experimental.
// apply to http header and grpc metadata
// dtm server will save these headers in trans creating request.
// and then passthrough them to sub-trans
func SetPassthroughHeaders(headers []string) {
dtmimp.PassthroughHeaders = headers
}

7
dtmcli/types_test.go

@ -10,8 +10,8 @@ import (
"net/url"
"testing"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/stretchr/testify/assert"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
func TestTypes(t *testing.T) {
@ -26,6 +26,7 @@ func TestTypes(t *testing.T) {
}
func TestXaSqlTimeout(t *testing.T) {
old := GetXaSqlTimeoutMs()
SetXaSqlTimeoutMs(old)
old := GetXaSQLTimeoutMs()
SetXaSQLTimeoutMs(old)
SetBarrierTableName(dtmimp.BarrierTableName) // just cover this func
}

18
dtmcli/xa.go

@ -11,8 +11,8 @@ import (
"fmt"
"net/url"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/go-resty/resty/v2"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
// XaGlobalFunc type of xa global function
@ -59,8 +59,8 @@ func NewXaClient(server string, mysqlConf DBConf, notifyURL string, register XaR
}
// HandleCallback 处理commit/rollback的回调
func (xc *XaClient) HandleCallback(gid string, branchID string, action string) (interface{}, error) {
return MapSuccess, xc.XaClientBase.HandleCallback(gid, branchID, action)
func (xc *XaClient) HandleCallback(gid string, branchID string, action string) interface{} {
return xc.XaClientBase.HandleCallback(gid, branchID, action)
}
// XaLocalTransaction start a xa local transaction
@ -83,11 +83,17 @@ func (xc *XaClient) XaLocalTransaction(qs url.Values, xaFunc XaLocalFunc) error
// XaGlobalTransaction start a xa global transaction
func (xc *XaClient) XaGlobalTransaction(gid string, xaFunc XaGlobalFunc) (rerr error) {
xa := Xa{TransBase: *dtmimp.NewTransBase(gid, "xa", xc.XaClientBase.Server, "")}
return xc.XaGlobalTransaction2(gid, func(x *Xa) {}, xaFunc)
}
// XaGlobalTransaction2 start a xa global transaction
func (xc *XaClient) XaGlobalTransaction2(gid string, custom func(*Xa), xaFunc XaGlobalFunc) (rerr error) {
xa := &Xa{TransBase: *dtmimp.NewTransBase(gid, "xa", xc.XaClientBase.Server, "")}
custom(xa)
return xc.HandleGlobalTrans(&xa.TransBase, func(action string) error {
return dtmimp.TransCallDtm(&xa.TransBase, &xa, action)
return dtmimp.TransCallDtm(&xa.TransBase, xa, action)
}, func() error {
_, rerr := xaFunc(&xa)
_, rerr := xaFunc(xa)
return rerr
})
}

4
dtmgrpc/barrier.go

@ -9,8 +9,8 @@ package dtmgrpc
import (
"context"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
)
// BarrierFromGrpc generate a Barrier from grpc context

2
dtmgrpc/dtmgimp/README.md

@ -0,0 +1,2 @@
## 注意
此包带imp后缀,主要被dtm内部使用,相关接口可能会发生变更,请勿使用这里的接口

510
dtmgrpc/dtmgimp/dtmgimp.pb.go

@ -1,510 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc v3.17.3
// source: dtmgrpc/dtmgimp/dtmgimp.proto
package dtmgimp
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
emptypb "google.golang.org/protobuf/types/known/emptypb"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type DtmTransOptions struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
WaitResult bool `protobuf:"varint,1,opt,name=WaitResult,proto3" json:"WaitResult,omitempty"`
TimeoutToFail int64 `protobuf:"varint,2,opt,name=TimeoutToFail,proto3" json:"TimeoutToFail,omitempty"`
RetryInterval int64 `protobuf:"varint,3,opt,name=RetryInterval,proto3" json:"RetryInterval,omitempty"`
}
func (x *DtmTransOptions) Reset() {
*x = DtmTransOptions{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmTransOptions) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmTransOptions) ProtoMessage() {}
func (x *DtmTransOptions) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmTransOptions.ProtoReflect.Descriptor instead.
func (*DtmTransOptions) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescGZIP(), []int{0}
}
func (x *DtmTransOptions) GetWaitResult() bool {
if x != nil {
return x.WaitResult
}
return false
}
func (x *DtmTransOptions) GetTimeoutToFail() int64 {
if x != nil {
return x.TimeoutToFail
}
return 0
}
func (x *DtmTransOptions) GetRetryInterval() int64 {
if x != nil {
return x.RetryInterval
}
return 0
}
// DtmRequest request sent to dtm server
type DtmRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Gid string `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
TransType string `protobuf:"bytes,2,opt,name=TransType,proto3" json:"TransType,omitempty"`
TransOptions *DtmTransOptions `protobuf:"bytes,3,opt,name=TransOptions,proto3" json:"TransOptions,omitempty"`
CustomedData string `protobuf:"bytes,4,opt,name=CustomedData,proto3" json:"CustomedData,omitempty"`
BinPayloads [][]byte `protobuf:"bytes,5,rep,name=BinPayloads,proto3" json:"BinPayloads,omitempty"` // for MSG/SAGA branch payloads
QueryPrepared string `protobuf:"bytes,6,opt,name=QueryPrepared,proto3" json:"QueryPrepared,omitempty"` // for MSG
Steps string `protobuf:"bytes,7,opt,name=Steps,proto3" json:"Steps,omitempty"`
}
func (x *DtmRequest) Reset() {
*x = DtmRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmRequest) ProtoMessage() {}
func (x *DtmRequest) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmRequest.ProtoReflect.Descriptor instead.
func (*DtmRequest) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescGZIP(), []int{1}
}
func (x *DtmRequest) GetGid() string {
if x != nil {
return x.Gid
}
return ""
}
func (x *DtmRequest) GetTransType() string {
if x != nil {
return x.TransType
}
return ""
}
func (x *DtmRequest) GetTransOptions() *DtmTransOptions {
if x != nil {
return x.TransOptions
}
return nil
}
func (x *DtmRequest) GetCustomedData() string {
if x != nil {
return x.CustomedData
}
return ""
}
func (x *DtmRequest) GetBinPayloads() [][]byte {
if x != nil {
return x.BinPayloads
}
return nil
}
func (x *DtmRequest) GetQueryPrepared() string {
if x != nil {
return x.QueryPrepared
}
return ""
}
func (x *DtmRequest) GetSteps() string {
if x != nil {
return x.Steps
}
return ""
}
type DtmGidReply struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Gid string `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
}
func (x *DtmGidReply) Reset() {
*x = DtmGidReply{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmGidReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmGidReply) ProtoMessage() {}
func (x *DtmGidReply) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmGidReply.ProtoReflect.Descriptor instead.
func (*DtmGidReply) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescGZIP(), []int{2}
}
func (x *DtmGidReply) GetGid() string {
if x != nil {
return x.Gid
}
return ""
}
type DtmBranchRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Gid string `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
TransType string `protobuf:"bytes,2,opt,name=TransType,proto3" json:"TransType,omitempty"`
BranchID string `protobuf:"bytes,3,opt,name=BranchID,proto3" json:"BranchID,omitempty"`
Op string `protobuf:"bytes,4,opt,name=Op,proto3" json:"Op,omitempty"`
Data map[string]string `protobuf:"bytes,5,rep,name=Data,proto3" json:"Data,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
BusiPayload []byte `protobuf:"bytes,6,opt,name=BusiPayload,proto3" json:"BusiPayload,omitempty"`
}
func (x *DtmBranchRequest) Reset() {
*x = DtmBranchRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmBranchRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmBranchRequest) ProtoMessage() {}
func (x *DtmBranchRequest) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmBranchRequest.ProtoReflect.Descriptor instead.
func (*DtmBranchRequest) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescGZIP(), []int{3}
}
func (x *DtmBranchRequest) GetGid() string {
if x != nil {
return x.Gid
}
return ""
}
func (x *DtmBranchRequest) GetTransType() string {
if x != nil {
return x.TransType
}
return ""
}
func (x *DtmBranchRequest) GetBranchID() string {
if x != nil {
return x.BranchID
}
return ""
}
func (x *DtmBranchRequest) GetOp() string {
if x != nil {
return x.Op
}
return ""
}
func (x *DtmBranchRequest) GetData() map[string]string {
if x != nil {
return x.Data
}
return nil
}
func (x *DtmBranchRequest) GetBusiPayload() []byte {
if x != nil {
return x.BusiPayload
}
return nil
}
var File_dtmgrpc_dtmgimp_dtmgimp_proto protoreflect.FileDescriptor
var file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDesc = []byte{
0x0a, 0x1d, 0x64, 0x74, 0x6d, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d,
0x70, 0x2f, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
0x07, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7d, 0x0a, 0x0f, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e,
0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x57, 0x61, 0x69, 0x74,
0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x57, 0x61,
0x69, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x69, 0x6d, 0x65,
0x6f, 0x75, 0x74, 0x54, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52,
0x0d, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x54, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x12, 0x24,
0x0a, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x79, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18,
0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x79, 0x49, 0x6e, 0x74, 0x65,
0x72, 0x76, 0x61, 0x6c, 0x22, 0xfc, 0x01, 0x0a, 0x0a, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x03, 0x47, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79,
0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54,
0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x70, 0x74, 0x69,
0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x64, 0x74, 0x6d, 0x67,
0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x70, 0x74, 0x69,
0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e,
0x73, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74,
0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x65,
0x64, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x42, 0x69, 0x6e, 0x50, 0x61, 0x79, 0x6c,
0x6f, 0x61, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, 0x42, 0x69, 0x6e, 0x50,
0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79,
0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d,
0x51, 0x75, 0x65, 0x72, 0x79, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x12, 0x14, 0x0a,
0x05, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x53, 0x74,
0x65, 0x70, 0x73, 0x22, 0x1f, 0x0a, 0x0b, 0x44, 0x74, 0x6d, 0x47, 0x69, 0x64, 0x52, 0x65, 0x70,
0x6c, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x03, 0x47, 0x69, 0x64, 0x22, 0x82, 0x02, 0x0a, 0x10, 0x44, 0x74, 0x6d, 0x42, 0x72, 0x61, 0x6e,
0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x54,
0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x42, 0x72, 0x61,
0x6e, 0x63, 0x68, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x42, 0x72, 0x61,
0x6e, 0x63, 0x68, 0x49, 0x44, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28,
0x09, 0x52, 0x02, 0x4f, 0x70, 0x12, 0x37, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74,
0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x44,
0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20,
0x0a, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x06, 0x20,
0x01, 0x28, 0x0c, 0x52, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
0x1a, 0x37, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xb1, 0x02, 0x0a, 0x03, 0x44, 0x74,
0x6d, 0x12, 0x38, 0x0a, 0x06, 0x4e, 0x65, 0x77, 0x47, 0x69, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x1a, 0x14, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74,
0x6d, 0x47, 0x69, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x06, 0x53,
0x75, 0x62, 0x6d, 0x69, 0x74, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e,
0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
0x74, 0x79, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x12,
0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36,
0x0a, 0x05, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d,
0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45,
0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74,
0x65, 0x72, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x12, 0x19, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69,
0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x42, 0x0b, 0x5a,
0x09, 0x2e, 0x2f, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescOnce sync.Once
file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescData = file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDesc
)
func file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescGZIP() []byte {
file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescOnce.Do(func() {
file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescData = protoimpl.X.CompressGZIP(file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescData)
})
return file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDescData
}
var file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_dtmgrpc_dtmgimp_dtmgimp_proto_goTypes = []interface{}{
(*DtmTransOptions)(nil), // 0: dtmgimp.DtmTransOptions
(*DtmRequest)(nil), // 1: dtmgimp.DtmRequest
(*DtmGidReply)(nil), // 2: dtmgimp.DtmGidReply
(*DtmBranchRequest)(nil), // 3: dtmgimp.DtmBranchRequest
nil, // 4: dtmgimp.DtmBranchRequest.DataEntry
(*emptypb.Empty)(nil), // 5: google.protobuf.Empty
}
var file_dtmgrpc_dtmgimp_dtmgimp_proto_depIdxs = []int32{
0, // 0: dtmgimp.DtmRequest.TransOptions:type_name -> dtmgimp.DtmTransOptions
4, // 1: dtmgimp.DtmBranchRequest.Data:type_name -> dtmgimp.DtmBranchRequest.DataEntry
5, // 2: dtmgimp.Dtm.NewGid:input_type -> google.protobuf.Empty
1, // 3: dtmgimp.Dtm.Submit:input_type -> dtmgimp.DtmRequest
1, // 4: dtmgimp.Dtm.Prepare:input_type -> dtmgimp.DtmRequest
1, // 5: dtmgimp.Dtm.Abort:input_type -> dtmgimp.DtmRequest
3, // 6: dtmgimp.Dtm.RegisterBranch:input_type -> dtmgimp.DtmBranchRequest
2, // 7: dtmgimp.Dtm.NewGid:output_type -> dtmgimp.DtmGidReply
5, // 8: dtmgimp.Dtm.Submit:output_type -> google.protobuf.Empty
5, // 9: dtmgimp.Dtm.Prepare:output_type -> google.protobuf.Empty
5, // 10: dtmgimp.Dtm.Abort:output_type -> google.protobuf.Empty
5, // 11: dtmgimp.Dtm.RegisterBranch:output_type -> google.protobuf.Empty
7, // [7:12] is the sub-list for method output_type
2, // [2:7] is the sub-list for method input_type
2, // [2:2] is the sub-list for extension type_name
2, // [2:2] is the sub-list for extension extendee
0, // [0:2] is the sub-list for field type_name
}
func init() { file_dtmgrpc_dtmgimp_dtmgimp_proto_init() }
func file_dtmgrpc_dtmgimp_dtmgimp_proto_init() {
if File_dtmgrpc_dtmgimp_dtmgimp_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmTransOptions); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmGidReply); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmBranchRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDesc,
NumEnums: 0,
NumMessages: 5,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_dtmgrpc_dtmgimp_dtmgimp_proto_goTypes,
DependencyIndexes: file_dtmgrpc_dtmgimp_dtmgimp_proto_depIdxs,
MessageInfos: file_dtmgrpc_dtmgimp_dtmgimp_proto_msgTypes,
}.Build()
File_dtmgrpc_dtmgimp_dtmgimp_proto = out.File
file_dtmgrpc_dtmgimp_dtmgimp_proto_rawDesc = nil
file_dtmgrpc_dtmgimp_dtmgimp_proto_goTypes = nil
file_dtmgrpc_dtmgimp_dtmgimp_proto_depIdxs = nil
}

33
dtmgrpc/dtmgimp/grpc_clients.go

@ -8,9 +8,13 @@ package dtmgimp
import (
"fmt"
"github.com/yedf/dtm/dtmcli/dtmimp"
grpc "google.golang.org/grpc"
"sync"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
grpc "google.golang.org/grpc"
)
type rawCodec struct{}
@ -22,9 +26,8 @@ func (cb rawCodec) Marshal(v interface{}) ([]byte, error) {
func (cb rawCodec) Unmarshal(data []byte, v interface{}) error {
ba, ok := v.(*[]byte)
dtmimp.PanicIf(!ok, fmt.Errorf("please pass in *[]byte"))
for _, byte := range data {
*ba = append(*ba, byte)
}
*ba = append(*ba, data...)
return nil
}
@ -32,14 +35,12 @@ func (cb rawCodec) Name() string { return "dtm_raw" }
var normalClients, rawClients sync.Map
// MustGetDtmClient 1
func MustGetDtmClient(grpcServer string) DtmClient {
return NewDtmClient(MustGetGrpcConn(grpcServer, false))
}
// ClientInterceptors declares grpc.UnaryClientInterceptors slice
var ClientInterceptors = []grpc.UnaryClientInterceptor{}
// MustGetRawDtmClient must get raw codec grpc conn
func MustGetRawDtmClient(grpcServer string) DtmClient {
return NewDtmClient(MustGetGrpcConn(grpcServer, true))
// MustGetDtmClient 1
func MustGetDtmClient(grpcServer string) dtmgpb.DtmClient {
return dtmgpb.NewDtmClient(MustGetGrpcConn(grpcServer, false))
}
// GetGrpcConn 1
@ -55,12 +56,14 @@ func GetGrpcConn(grpcServer string, isRaw bool) (conn *grpc.ClientConn, rerr err
if isRaw {
opts = grpc.WithDefaultCallOptions(grpc.ForceCodec(rawCodec{}))
}
dtmimp.Logf("grpc client connecting %s", grpcServer)
conn, rerr := grpc.Dial(grpcServer, grpc.WithInsecure(), grpc.WithUnaryInterceptor(GrpcClientLog), opts)
logger.Debugf("grpc client connecting %s", grpcServer)
interceptors := append(ClientInterceptors, GrpcClientLog)
inOpt := grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(interceptors...))
conn, rerr := grpc.Dial(grpcServer, inOpt, grpc.WithInsecure(), opts)
if rerr == nil {
clients.Store(grpcServer, conn)
v = conn
dtmimp.Logf("grpc client inited for %s", grpcServer)
logger.Debugf("grpc client inited for %s", grpcServer)
}
}
return v.(*grpc.ClientConn), rerr

40
dtmgrpc/dtmgimp/types.go

@ -9,50 +9,40 @@ package dtmgimp
import (
"context"
"fmt"
"time"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// GrpcServerLog 打印grpc服务端的日志
func GrpcServerLog(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
dtmimp.Logf("grpc server handling: %s %v", info.FullMethod, req)
began := time.Now()
logger.Debugf("grpc server handling: %s %s", info.FullMethod, dtmimp.MustMarshalString(req))
LogDtmCtx(ctx)
m, err := handler(ctx, req)
res := fmt.Sprintf("grpc server handled: %s %v result: %v err: %v", info.FullMethod, req, m, err)
res := fmt.Sprintf("%2dms %v %s %s %s",
time.Since(began).Milliseconds(), err, info.FullMethod, dtmimp.MustMarshalString(m), dtmimp.MustMarshalString(req))
if err != nil {
dtmimp.LogRedf("%s", res)
logger.Errorf("%s", res)
} else {
dtmimp.Logf("%s", res)
logger.Infof("%s", res)
}
return m, err
}
// GrpcClientLog 打印grpc服务端的日志
// GrpcClientLog 打印grpc调用的日志
func GrpcClientLog(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
dtmimp.Logf("grpc client calling: %s%s %v", cc.Target(), method, req)
logger.Debugf("grpc client calling: %s%s %v", cc.Target(), method, dtmimp.MustMarshalString(req))
LogDtmCtx(ctx)
err := invoker(ctx, method, req, reply, cc, opts...)
res := fmt.Sprintf("grpc client called: %s%s %v result: %v err: %v", cc.Target(), method, req, reply, err)
res := fmt.Sprintf("grpc client called: %s%s %s result: %s err: %v",
cc.Target(), method, dtmimp.MustMarshalString(req), dtmimp.MustMarshalString(reply), err)
if err != nil {
dtmimp.LogRedf("%s", res)
logger.Errorf("%s", res)
} else {
dtmimp.Logf("%s", res)
logger.Debugf("%s", res)
}
return err
}
// Result2Error 将通用的result转成grpc的error
func Result2Error(res interface{}, err error) error {
e := dtmimp.CheckResult(res, err)
if e == dtmimp.ErrFailure {
dtmimp.LogRedf("failure: res: %v, err: %v", res, e)
return status.New(codes.Aborted, dtmcli.ResultFailure).Err()
} else if e == dtmimp.ErrOngoing {
return status.New(codes.Aborted, dtmcli.ResultOngoing).Err()
}
return e
}

55
dtmgrpc/dtmgimp/utils.go

@ -9,7 +9,9 @@ package dtmgimp
import (
context "context"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
"google.golang.org/grpc/metadata"
"google.golang.org/protobuf/proto"
emptypb "google.golang.org/protobuf/types/known/emptypb"
@ -25,13 +27,15 @@ func MustProtoMarshal(msg proto.Message) []byte {
// DtmGrpcCall make a convenient call to dtm
func DtmGrpcCall(s *dtmimp.TransBase, operation string) error {
reply := emptypb.Empty{}
return MustGetGrpcConn(s.Dtm, false).Invoke(context.Background(), "/dtmgimp.Dtm/"+operation, &DtmRequest{
return MustGetGrpcConn(s.Dtm, false).Invoke(context.Background(), "/dtmgimp.Dtm/"+operation, &dtmgpb.DtmRequest{
Gid: s.Gid,
TransType: s.TransType,
TransOptions: &DtmTransOptions{
WaitResult: s.WaitResult,
TimeoutToFail: s.TimeoutToFail,
RetryInterval: s.RetryInterval,
TransOptions: &dtmgpb.DtmTransOptions{
WaitResult: s.WaitResult,
TimeoutToFail: s.TimeoutToFail,
RetryInterval: s.RetryInterval,
PassthroughHeaders: s.PassthroughHeaders,
BranchHeaders: s.BranchHeaders,
},
QueryPrepared: s.QueryPrepared,
CustomedData: s.CustomData,
@ -40,30 +44,43 @@ func DtmGrpcCall(s *dtmimp.TransBase, operation string) error {
}, &reply)
}
const mdpre string = "dtm-"
const dtmpre string = "dtm-"
// TransInfo2Ctx add trans info to grpc context
func TransInfo2Ctx(gid, transType, branchID, op, dtm string) context.Context {
md := metadata.Pairs(
mdpre+"gid", gid,
mdpre+"trans_type", transType,
mdpre+"branch_id", branchID,
mdpre+"op", op,
mdpre+"dtm", dtm,
dtmpre+"gid", gid,
dtmpre+"trans_type", transType,
dtmpre+"branch_id", branchID,
dtmpre+"op", op,
dtmpre+"dtm", dtm,
)
return metadata.NewOutgoingContext(context.Background(), md)
}
// Map2Kvs map to metadata kv
func Map2Kvs(m map[string]string) []string {
kvs := []string{}
for k, v := range m {
kvs = append(kvs, k, v)
}
return kvs
}
// LogDtmCtx logout dtm info in context metadata
func LogDtmCtx(ctx context.Context) {
tb := TransBaseFromGrpc(ctx)
if tb.Gid != "" {
dtmimp.Logf("gid: %s trans_type: %s branch_id: %s op: %s dtm: %s", tb.Gid, tb.TransType, tb.BranchID, tb.Op, tb.Dtm)
logger.Debugf("gid: %s trans_type: %s branch_id: %s op: %s dtm: %s", tb.Gid, tb.TransType, tb.BranchID, tb.Op, tb.Dtm)
}
}
func dtmGet(md metadata.MD, key string) string {
return mdGet(md, dtmpre+key)
}
func mdGet(md metadata.MD, key string) string {
v := md.Get(mdpre + key)
v := md.Get(key)
if len(v) == 0 {
return ""
}
@ -73,7 +90,13 @@ func mdGet(md metadata.MD, key string) string {
// TransBaseFromGrpc get trans base info from a context metadata
func TransBaseFromGrpc(ctx context.Context) *dtmimp.TransBase {
md, _ := metadata.FromIncomingContext(ctx)
tb := dtmimp.NewTransBase(mdGet(md, "gid"), mdGet(md, "trans_type"), mdGet(md, "dtm"), mdGet(md, "branch_id"))
tb.Op = mdGet(md, "op")
tb := dtmimp.NewTransBase(dtmGet(md, "gid"), dtmGet(md, "trans_type"), dtmGet(md, "dtm"), dtmGet(md, "branch_id"))
tb.Op = dtmGet(md, "op")
return tb
}
// GetMetaFromContext get header from context
func GetMetaFromContext(ctx context.Context, name string) string {
md, _ := metadata.FromIncomingContext(ctx)
return mdGet(md, name)
}

534
dtmgrpc/dtmgpb/dtmgimp.pb.go

@ -0,0 +1,534 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc v3.17.3
// source: dtmgrpc/dtmgpb/dtmgimp.proto
package dtmgpb
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
emptypb "google.golang.org/protobuf/types/known/emptypb"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type DtmTransOptions struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
WaitResult bool `protobuf:"varint,1,opt,name=WaitResult,proto3" json:"WaitResult,omitempty"`
TimeoutToFail int64 `protobuf:"varint,2,opt,name=TimeoutToFail,proto3" json:"TimeoutToFail,omitempty"`
RetryInterval int64 `protobuf:"varint,3,opt,name=RetryInterval,proto3" json:"RetryInterval,omitempty"`
PassthroughHeaders []string `protobuf:"bytes,4,rep,name=PassthroughHeaders,proto3" json:"PassthroughHeaders,omitempty"`
BranchHeaders map[string]string `protobuf:"bytes,5,rep,name=BranchHeaders,proto3" json:"BranchHeaders,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (x *DtmTransOptions) Reset() {
*x = DtmTransOptions{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmTransOptions) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmTransOptions) ProtoMessage() {}
func (x *DtmTransOptions) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmTransOptions.ProtoReflect.Descriptor instead.
func (*DtmTransOptions) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP(), []int{0}
}
func (x *DtmTransOptions) GetWaitResult() bool {
if x != nil {
return x.WaitResult
}
return false
}
func (x *DtmTransOptions) GetTimeoutToFail() int64 {
if x != nil {
return x.TimeoutToFail
}
return 0
}
func (x *DtmTransOptions) GetRetryInterval() int64 {
if x != nil {
return x.RetryInterval
}
return 0
}
func (x *DtmTransOptions) GetPassthroughHeaders() []string {
if x != nil {
return x.PassthroughHeaders
}
return nil
}
func (x *DtmTransOptions) GetBranchHeaders() map[string]string {
if x != nil {
return x.BranchHeaders
}
return nil
}
// DtmRequest request sent to dtm server
type DtmRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Gid string `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
TransType string `protobuf:"bytes,2,opt,name=TransType,proto3" json:"TransType,omitempty"`
TransOptions *DtmTransOptions `protobuf:"bytes,3,opt,name=TransOptions,proto3" json:"TransOptions,omitempty"`
CustomedData string `protobuf:"bytes,4,opt,name=CustomedData,proto3" json:"CustomedData,omitempty"`
BinPayloads [][]byte `protobuf:"bytes,5,rep,name=BinPayloads,proto3" json:"BinPayloads,omitempty"` // for MSG/SAGA branch payloads
QueryPrepared string `protobuf:"bytes,6,opt,name=QueryPrepared,proto3" json:"QueryPrepared,omitempty"` // for MSG
Steps string `protobuf:"bytes,7,opt,name=Steps,proto3" json:"Steps,omitempty"`
}
func (x *DtmRequest) Reset() {
*x = DtmRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmRequest) ProtoMessage() {}
func (x *DtmRequest) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmRequest.ProtoReflect.Descriptor instead.
func (*DtmRequest) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP(), []int{1}
}
func (x *DtmRequest) GetGid() string {
if x != nil {
return x.Gid
}
return ""
}
func (x *DtmRequest) GetTransType() string {
if x != nil {
return x.TransType
}
return ""
}
func (x *DtmRequest) GetTransOptions() *DtmTransOptions {
if x != nil {
return x.TransOptions
}
return nil
}
func (x *DtmRequest) GetCustomedData() string {
if x != nil {
return x.CustomedData
}
return ""
}
func (x *DtmRequest) GetBinPayloads() [][]byte {
if x != nil {
return x.BinPayloads
}
return nil
}
func (x *DtmRequest) GetQueryPrepared() string {
if x != nil {
return x.QueryPrepared
}
return ""
}
func (x *DtmRequest) GetSteps() string {
if x != nil {
return x.Steps
}
return ""
}
type DtmGidReply struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Gid string `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
}
func (x *DtmGidReply) Reset() {
*x = DtmGidReply{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmGidReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmGidReply) ProtoMessage() {}
func (x *DtmGidReply) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmGidReply.ProtoReflect.Descriptor instead.
func (*DtmGidReply) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP(), []int{2}
}
func (x *DtmGidReply) GetGid() string {
if x != nil {
return x.Gid
}
return ""
}
type DtmBranchRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Gid string `protobuf:"bytes,1,opt,name=Gid,proto3" json:"Gid,omitempty"`
TransType string `protobuf:"bytes,2,opt,name=TransType,proto3" json:"TransType,omitempty"`
BranchID string `protobuf:"bytes,3,opt,name=BranchID,proto3" json:"BranchID,omitempty"`
Op string `protobuf:"bytes,4,opt,name=Op,proto3" json:"Op,omitempty"`
Data map[string]string `protobuf:"bytes,5,rep,name=Data,proto3" json:"Data,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
BusiPayload []byte `protobuf:"bytes,6,opt,name=BusiPayload,proto3" json:"BusiPayload,omitempty"`
}
func (x *DtmBranchRequest) Reset() {
*x = DtmBranchRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DtmBranchRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DtmBranchRequest) ProtoMessage() {}
func (x *DtmBranchRequest) ProtoReflect() protoreflect.Message {
mi := &file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DtmBranchRequest.ProtoReflect.Descriptor instead.
func (*DtmBranchRequest) Descriptor() ([]byte, []int) {
return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP(), []int{3}
}
func (x *DtmBranchRequest) GetGid() string {
if x != nil {
return x.Gid
}
return ""
}
func (x *DtmBranchRequest) GetTransType() string {
if x != nil {
return x.TransType
}
return ""
}
func (x *DtmBranchRequest) GetBranchID() string {
if x != nil {
return x.BranchID
}
return ""
}
func (x *DtmBranchRequest) GetOp() string {
if x != nil {
return x.Op
}
return ""
}
func (x *DtmBranchRequest) GetData() map[string]string {
if x != nil {
return x.Data
}
return nil
}
func (x *DtmBranchRequest) GetBusiPayload() []byte {
if x != nil {
return x.BusiPayload
}
return nil
}
var File_dtmgrpc_dtmgpb_dtmgimp_proto protoreflect.FileDescriptor
var file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc = []byte{
0x0a, 0x1c, 0x64, 0x74, 0x6d, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x74, 0x6d, 0x67, 0x70, 0x62,
0x2f, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07,
0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc2, 0x02, 0x0a, 0x0f, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e,
0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x57, 0x61, 0x69, 0x74,
0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x57, 0x61,
0x69, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x69, 0x6d, 0x65,
0x6f, 0x75, 0x74, 0x54, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52,
0x0d, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x54, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x12, 0x24,
0x0a, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x79, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18,
0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x79, 0x49, 0x6e, 0x74, 0x65,
0x72, 0x76, 0x61, 0x6c, 0x12, 0x2e, 0x0a, 0x12, 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f,
0x75, 0x67, 0x68, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09,
0x52, 0x12, 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x48, 0x65, 0x61,
0x64, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, 0x0d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x48, 0x65,
0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x64, 0x74,
0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x70,
0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x48, 0x65, 0x61, 0x64,
0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68,
0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x42, 0x72, 0x61, 0x6e, 0x63,
0x68, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfc, 0x01, 0x0a, 0x0a, 0x44, 0x74,
0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x54, 0x72,
0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x54,
0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e,
0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18,
0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x54, 0x72, 0x61, 0x6e,
0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d,
0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x75,
0x73, 0x74, 0x6f, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x42, 0x69,
0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52,
0x0b, 0x42, 0x69, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x24, 0x0a, 0x0d,
0x51, 0x75, 0x65, 0x72, 0x79, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20,
0x01, 0x28, 0x09, 0x52, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72,
0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28,
0x09, 0x52, 0x05, 0x53, 0x74, 0x65, 0x70, 0x73, 0x22, 0x1f, 0x0a, 0x0b, 0x44, 0x74, 0x6d, 0x47,
0x69, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64, 0x22, 0x82, 0x02, 0x0a, 0x10, 0x44, 0x74,
0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10,
0x0a, 0x03, 0x47, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x47, 0x69, 0x64,
0x12, 0x1c, 0x0a, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a,
0x0a, 0x08, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x49, 0x44, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x70,
0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x4f, 0x70, 0x12, 0x37, 0x0a, 0x04, 0x44, 0x61,
0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69,
0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x44,
0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61, 0x79, 0x6c, 0x6f,
0x61, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x42, 0x75, 0x73, 0x69, 0x50, 0x61,
0x79, 0x6c, 0x6f, 0x61, 0x64, 0x1a, 0x37, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74,
0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xb1,
0x02, 0x0a, 0x03, 0x44, 0x74, 0x6d, 0x12, 0x38, 0x0a, 0x06, 0x4e, 0x65, 0x77, 0x47, 0x69, 0x64,
0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x14, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69,
0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x47, 0x69, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00,
0x12, 0x37, 0x0a, 0x06, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d,
0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x07, 0x50, 0x72, 0x65,
0x70, 0x61, 0x72, 0x65, 0x12, 0x13, 0x2e, 0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44,
0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x05, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x12, 0x13, 0x2e, 0x64,
0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0e, 0x52,
0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x12, 0x19, 0x2e,
0x64, 0x74, 0x6d, 0x67, 0x69, 0x6d, 0x70, 0x2e, 0x44, 0x74, 0x6d, 0x42, 0x72, 0x61, 0x6e, 0x63,
0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x2f, 0x64, 0x74, 0x6d, 0x67, 0x70, 0x62, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescOnce sync.Once
file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescData = file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc
)
func file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescGZIP() []byte {
file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescOnce.Do(func() {
file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescData = protoimpl.X.CompressGZIP(file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescData)
})
return file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDescData
}
var file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
var file_dtmgrpc_dtmgpb_dtmgimp_proto_goTypes = []interface{}{
(*DtmTransOptions)(nil), // 0: dtmgimp.DtmTransOptions
(*DtmRequest)(nil), // 1: dtmgimp.DtmRequest
(*DtmGidReply)(nil), // 2: dtmgimp.DtmGidReply
(*DtmBranchRequest)(nil), // 3: dtmgimp.DtmBranchRequest
nil, // 4: dtmgimp.DtmTransOptions.BranchHeadersEntry
nil, // 5: dtmgimp.DtmBranchRequest.DataEntry
(*emptypb.Empty)(nil), // 6: google.protobuf.Empty
}
var file_dtmgrpc_dtmgpb_dtmgimp_proto_depIdxs = []int32{
4, // 0: dtmgimp.DtmTransOptions.BranchHeaders:type_name -> dtmgimp.DtmTransOptions.BranchHeadersEntry
0, // 1: dtmgimp.DtmRequest.TransOptions:type_name -> dtmgimp.DtmTransOptions
5, // 2: dtmgimp.DtmBranchRequest.Data:type_name -> dtmgimp.DtmBranchRequest.DataEntry
6, // 3: dtmgimp.Dtm.NewGid:input_type -> google.protobuf.Empty
1, // 4: dtmgimp.Dtm.Submit:input_type -> dtmgimp.DtmRequest
1, // 5: dtmgimp.Dtm.Prepare:input_type -> dtmgimp.DtmRequest
1, // 6: dtmgimp.Dtm.Abort:input_type -> dtmgimp.DtmRequest
3, // 7: dtmgimp.Dtm.RegisterBranch:input_type -> dtmgimp.DtmBranchRequest
2, // 8: dtmgimp.Dtm.NewGid:output_type -> dtmgimp.DtmGidReply
6, // 9: dtmgimp.Dtm.Submit:output_type -> google.protobuf.Empty
6, // 10: dtmgimp.Dtm.Prepare:output_type -> google.protobuf.Empty
6, // 11: dtmgimp.Dtm.Abort:output_type -> google.protobuf.Empty
6, // 12: dtmgimp.Dtm.RegisterBranch:output_type -> google.protobuf.Empty
8, // [8:13] is the sub-list for method output_type
3, // [3:8] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
}
func init() { file_dtmgrpc_dtmgpb_dtmgimp_proto_init() }
func file_dtmgrpc_dtmgpb_dtmgimp_proto_init() {
if File_dtmgrpc_dtmgpb_dtmgimp_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmTransOptions); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmGidReply); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DtmBranchRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc,
NumEnums: 0,
NumMessages: 6,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_dtmgrpc_dtmgpb_dtmgimp_proto_goTypes,
DependencyIndexes: file_dtmgrpc_dtmgpb_dtmgimp_proto_depIdxs,
MessageInfos: file_dtmgrpc_dtmgpb_dtmgimp_proto_msgTypes,
}.Build()
File_dtmgrpc_dtmgpb_dtmgimp_proto = out.File
file_dtmgrpc_dtmgpb_dtmgimp_proto_rawDesc = nil
file_dtmgrpc_dtmgpb_dtmgimp_proto_goTypes = nil
file_dtmgrpc_dtmgpb_dtmgimp_proto_depIdxs = nil
}

4
dtmgrpc/dtmgimp/dtmgimp.proto → dtmgrpc/dtmgpb/dtmgimp.proto

@ -1,6 +1,6 @@
syntax = "proto3";
option go_package = "./dtmgimp";
option go_package = "./dtmgpb";
import "google/protobuf/empty.proto";
package dtmgimp;
@ -18,6 +18,8 @@ message DtmTransOptions {
bool WaitResult = 1;
int64 TimeoutToFail = 2;
int64 RetryInterval = 3;
repeated string PassthroughHeaders = 4;
map<string, string> BranchHeaders = 5;
}
// DtmRequest request sent to dtm server

10
dtmgrpc/dtmgimp/dtmgimp_grpc.pb.go → dtmgrpc/dtmgpb/dtmgimp_grpc.pb.go

@ -1,12 +1,6 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
package dtmgimp
package dtmgpb
import (
context "context"
@ -248,5 +242,5 @@ var Dtm_ServiceDesc = grpc.ServiceDesc{
},
},
Streams: []grpc.StreamDesc{},
Metadata: "dtmgrpc/dtmgimp/dtmgimp.proto",
Metadata: "dtmgrpc/dtmgpb/dtmgimp.proto",
}

26
dtmgrpc/msg.go

@ -7,9 +7,11 @@
package dtmgrpc
import (
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"database/sql"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"google.golang.org/protobuf/proto"
)
@ -40,3 +42,21 @@ func (s *MsgGrpc) Prepare(queryPrepared string) error {
func (s *MsgGrpc) Submit() error {
return dtmgimp.DtmGrpcCall(&s.TransBase, "Submit")
}
// PrepareAndSubmit one method for the entire busi->prepare->submit
func (s *MsgGrpc) PrepareAndSubmit(queryPrepared string, db *sql.DB, busiCall dtmcli.BarrierBusiFunc) error {
bb, err := dtmcli.BarrierFrom(s.TransType, s.Gid, "00", "msg") // a special barrier for msg QueryPrepared
if err == nil {
err = bb.CallWithDB(db, func(tx *sql.Tx) error {
err := busiCall(tx)
if err == nil {
err = s.Prepare(queryPrepared)
}
return err
})
}
if err == nil {
err = s.Submit()
}
return err
}

4
dtmgrpc/saga.go

@ -7,8 +7,8 @@
package dtmgrpc
import (
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"google.golang.org/protobuf/proto"
)

46
dtmgrpc/tcc.go

@ -10,9 +10,11 @@ import (
context "context"
"fmt"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/yedf/dtmdriver"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
"github.com/dtm-labs/dtmdriver"
"google.golang.org/grpc/metadata"
"google.golang.org/protobuf/proto"
)
@ -29,13 +31,14 @@ type TccGlobalFunc func(tcc *TccGrpc) error
// gid 全局事务id
// tccFunc tcc事务函数,里面会定义全局事务的分支
func TccGlobalTransaction(dtm string, gid string, tccFunc TccGlobalFunc) (rerr error) {
return TccGlobalTransaction2(dtm, gid, func(tg *TccGrpc) {}, tccFunc)
}
// TccGlobalTransaction2 new version of TccGlobalTransaction
func TccGlobalTransaction2(dtm string, gid string, custom func(*TccGrpc), tccFunc TccGlobalFunc) (rerr error) {
tcc := &TccGrpc{TransBase: *dtmimp.NewTransBase(gid, "tcc", dtm, "")}
dc := dtmgimp.MustGetDtmClient(tcc.Dtm)
dr := &dtmgimp.DtmRequest{
Gid: tcc.Gid,
TransType: tcc.TransType,
}
_, rerr = dc.Prepare(context.Background(), dr)
custom(tcc)
rerr = dtmgimp.DtmGrpcCall(&tcc.TransBase, "Prepare")
if rerr != nil {
return rerr
}
@ -43,10 +46,10 @@ func TccGlobalTransaction(dtm string, gid string, tccFunc TccGlobalFunc) (rerr e
defer func() {
x := recover()
if x == nil && rerr == nil {
_, rerr = dc.Submit(context.Background(), dr)
rerr = dtmgimp.DtmGrpcCall(&tcc.TransBase, "Submit")
return
}
_, err := dc.Abort(context.Background(), dr)
err := dtmgimp.DtmGrpcCall(&tcc.TransBase, "Abort")
if rerr == nil {
rerr = err
}
@ -72,13 +75,15 @@ func TccFromGrpc(ctx context.Context) (*TccGrpc, error) {
func (t *TccGrpc) CallBranch(busiMsg proto.Message, tryURL string, confirmURL string, cancelURL string, reply interface{}) error {
branchID := t.NewSubBranchID()
bd, err := proto.Marshal(busiMsg)
_, err = dtmgimp.MustGetDtmClient(t.Dtm).RegisterBranch(context.Background(), &dtmgimp.DtmBranchRequest{
Gid: t.Gid,
TransType: t.TransType,
BranchID: branchID,
BusiPayload: bd,
Data: map[string]string{"confirm": confirmURL, "cancel": cancelURL},
})
if err == nil {
_, err = dtmgimp.MustGetDtmClient(t.Dtm).RegisterBranch(context.Background(), &dtmgpb.DtmBranchRequest{
Gid: t.Gid,
TransType: t.TransType,
BranchID: branchID,
BusiPayload: bd,
Data: map[string]string{"confirm": confirmURL, "cancel": cancelURL},
})
}
if err != nil {
return err
}
@ -86,6 +91,7 @@ func (t *TccGrpc) CallBranch(busiMsg proto.Message, tryURL string, confirmURL st
if err != nil {
return err
}
return dtmgimp.MustGetGrpcConn(server, false).Invoke(
dtmgimp.TransInfo2Ctx(t.Gid, t.TransType, branchID, "try", t.Dtm), method, busiMsg, reply)
ctx := dtmgimp.TransInfo2Ctx(t.Gid, t.TransType, branchID, "try", t.Dtm)
ctx = metadata.AppendToOutgoingContext(ctx, dtmgimp.Map2Kvs(t.BranchHeaders)...)
return dtmgimp.MustGetGrpcConn(server, false).Invoke(ctx, method, busiMsg, reply)
}

38
dtmgrpc/type.go

@ -9,13 +9,27 @@ package dtmgrpc
import (
context "context"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/yedf/dtmdriver"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtmdriver"
grpc "google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
emptypb "google.golang.org/protobuf/types/known/emptypb"
)
// DtmError2GrpcError translate dtm error to grpc error
func DtmError2GrpcError(res interface{}) error {
e, ok := res.(error)
if ok && e == dtmimp.ErrFailure {
return status.New(codes.Aborted, dtmcli.ResultFailure).Err()
} else if ok && e == dtmimp.ErrOngoing {
return status.New(codes.FailedPrecondition, dtmcli.ResultOngoing).Err()
}
return e
}
// MustGenGid must gen a gid from grpcServer
func MustGenGid(grpcServer string) string {
dc := dtmgimp.MustGetDtmClient(grpcServer)
@ -24,16 +38,12 @@ func MustGenGid(grpcServer string) string {
return r.Gid
}
// SetCurrentDBType set the current db type
func SetCurrentDBType(dbType string) {
dtmcli.SetCurrentDBType(dbType)
}
// GetCurrentDBType set the current db type
func GetCurrentDBType() string {
return dtmcli.GetCurrentDBType()
}
// UseDriver use the specified driver to handle grpc urls
func UseDriver(driverName string) error {
return dtmdriver.Use(driverName)
}
// AddUnaryInterceptor adds grpc.UnaryClientInterceptor
func AddUnaryInterceptor(interceptor grpc.UnaryClientInterceptor) {
dtmgimp.ClientInterceptors = append(dtmgimp.ClientInterceptors, interceptor)
}

5
dtmgrpc/type_test.go

@ -11,7 +11,6 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/yedf/dtm/dtmcli"
)
func TestType(t *testing.T) {
@ -21,10 +20,6 @@ func TestType(t *testing.T) {
_, err = TccFromGrpc(context.Background())
assert.Error(t, err)
old := GetCurrentDBType()
SetCurrentDBType(dtmcli.DBTypeMysql)
SetCurrentDBType(old)
err = UseDriver("default")
assert.Nil(t, err)
}

25
dtmgrpc/xa.go

@ -11,10 +11,11 @@ import (
"database/sql"
"fmt"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/yedf/dtmdriver"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
"github.com/dtm-labs/dtmdriver"
grpc "google.golang.org/grpc"
"google.golang.org/protobuf/proto"
emptypb "google.golang.org/protobuf/types/known/emptypb"
@ -78,7 +79,7 @@ func (xc *XaGrpcClient) XaLocalTransaction(ctx context.Context, msg proto.Messag
if err != nil {
return err
}
_, err = dtmgimp.MustGetDtmClient(xa.Dtm).RegisterBranch(context.Background(), &dtmgimp.DtmBranchRequest{
_, err = dtmgimp.MustGetDtmClient(xa.Dtm).RegisterBranch(context.Background(), &dtmgpb.DtmBranchRequest{
Gid: xa.Gid,
BranchID: xa.BranchID,
TransType: xa.TransType,
@ -91,14 +92,20 @@ func (xc *XaGrpcClient) XaLocalTransaction(ctx context.Context, msg proto.Messag
// XaGlobalTransaction start a xa global transaction
func (xc *XaGrpcClient) XaGlobalTransaction(gid string, xaFunc XaGrpcGlobalFunc) error {
xa := XaGrpc{TransBase: *dtmimp.NewTransBase(gid, "xa", xc.Server, "")}
return xc.XaGlobalTransaction2(gid, func(xg *XaGrpc) {}, xaFunc)
}
// XaGlobalTransaction2 new version of XaGlobalTransaction. support custom
func (xc *XaGrpcClient) XaGlobalTransaction2(gid string, custom func(*XaGrpc), xaFunc XaGrpcGlobalFunc) error {
xa := &XaGrpc{TransBase: *dtmimp.NewTransBase(gid, "xa", xc.Server, "")}
custom(xa)
dc := dtmgimp.MustGetDtmClient(xa.Dtm)
req := &dtmgimp.DtmRequest{
req := &dtmgpb.DtmRequest{
Gid: gid,
TransType: xa.TransType,
}
return xc.HandleGlobalTrans(&xa.TransBase, func(action string) error {
f := map[string]func(context.Context, *dtmgimp.DtmRequest, ...grpc.CallOption) (*emptypb.Empty, error){
f := map[string]func(context.Context, *dtmgpb.DtmRequest, ...grpc.CallOption) (*emptypb.Empty, error){
"prepare": dc.Prepare,
"submit": dc.Submit,
"abort": dc.Abort,
@ -106,7 +113,7 @@ func (xc *XaGrpcClient) XaGlobalTransaction(gid string, xaFunc XaGrpcGlobalFunc)
_, err := f(context.Background(), req)
return err
}, func() error {
return xaFunc(&xa)
return xaFunc(xa)
})
}

47
dtmsvr/api.go

@ -9,48 +9,56 @@ package dtmsvr
import (
"fmt"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmsvr/storage"
)
func svcSubmit(t *TransGlobal) (interface{}, error) {
func svcSubmit(t *TransGlobal) interface{} {
t.Status = dtmcli.StatusSubmitted
err := t.saveNew()
branches, err := t.saveNew()
if err == storage.ErrUniqueConflict {
dbt := GetTransGlobal(t.Gid)
if dbt.Status == dtmcli.StatusPrepared {
dbt.changeStatus(t.Status)
branches = GetStore().FindBranches(t.Gid)
} else if dbt.Status != dtmcli.StatusSubmitted {
return map[string]interface{}{"dtm_result": dtmcli.ResultFailure, "message": fmt.Sprintf("current status '%s', cannot sumbmit", dbt.Status)}, nil
return fmt.Errorf("current status '%s', cannot sumbmit. %w", dbt.Status, dtmcli.ErrFailure)
}
}
return t.Process(), nil
return t.Process(branches)
}
func svcPrepare(t *TransGlobal) (interface{}, error) {
func svcPrepare(t *TransGlobal) interface{} {
t.Status = dtmcli.StatusPrepared
err := t.saveNew()
_, err := t.saveNew()
if err == storage.ErrUniqueConflict {
dbt := GetTransGlobal(t.Gid)
if dbt.Status != dtmcli.StatusPrepared {
return map[string]interface{}{"dtm_result": dtmcli.ResultFailure, "message": fmt.Sprintf("current status '%s', cannot prepare", dbt.Status)}, nil
return fmt.Errorf("current status '%s', cannot prepare. %w", dbt.Status, dtmcli.ErrFailure)
}
return nil
}
return dtmcli.MapSuccess, nil
return err
}
func svcAbort(t *TransGlobal) (interface{}, error) {
func svcAbort(t *TransGlobal) interface{} {
dbt := GetTransGlobal(t.Gid)
if dbt.TransType == "msg" && dbt.Status == dtmcli.StatusPrepared {
dbt.changeStatus(dtmcli.StatusFailed)
return nil
}
if t.TransType != "xa" && t.TransType != "tcc" || dbt.Status != dtmcli.StatusPrepared && dbt.Status != dtmcli.StatusAborting {
return map[string]interface{}{"dtm_result": dtmcli.ResultFailure, "message": fmt.Sprintf("trans type: '%s' current status '%s', cannot abort", dbt.TransType, dbt.Status)}, nil
return fmt.Errorf("trans type: '%s' current status '%s', cannot abort. %w", dbt.TransType, dbt.Status, dtmcli.ErrFailure)
}
dbt.changeStatus(dtmcli.StatusAborting)
return dbt.Process(), nil
branches := GetStore().FindBranches(t.Gid)
return dbt.Process(branches)
}
func svcRegisterBranch(transType string, branch *TransBranch, data map[string]string) (ret interface{}, rerr error) {
func svcRegisterBranch(transType string, branch *TransBranch, data map[string]string) error {
branches := []TransBranch{*branch, *branch}
if transType == "tcc" {
for i, b := range []string{dtmcli.BranchCancel, dtmcli.BranchConfirm} {
@ -63,7 +71,7 @@ func svcRegisterBranch(transType string, branch *TransBranch, data map[string]st
branches[1].Op = dtmcli.BranchCommit
branches[1].URL = data["url"]
} else {
return nil, fmt.Errorf("unknow trans type: %s", transType)
return fmt.Errorf("unknow trans type: %s", transType)
}
err := dtmimp.CatchP(func() {
@ -71,7 +79,10 @@ func svcRegisterBranch(transType string, branch *TransBranch, data map[string]st
})
if err == storage.ErrNotFound {
msg := fmt.Sprintf("no trans with gid: %s status: %s found", branch.Gid, dtmcli.StatusPrepared)
return map[string]interface{}{"dtm_result": dtmcli.ResultFailure, "message": msg}, nil
logger.Errorf(msg)
return fmt.Errorf("message: %s %w", msg, dtmcli.ErrFailure)
}
return dtmimp.If(err != nil, nil, dtmcli.MapSuccess), err
logger.Infof("LockGlobalSaveBranches result: %v: gid: %s old status: %s branches: %s",
err, branch.Gid, dtmcli.StatusPrepared, dtmimp.MustMarshalString(branches))
return err
}

26
dtmsvr/api_grpc.go

@ -9,9 +9,9 @@ package dtmsvr
import (
"context"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
pb "github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmgrpc"
pb "github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
"google.golang.org/protobuf/types/known/emptypb"
)
@ -20,31 +20,31 @@ type dtmServer struct {
pb.UnimplementedDtmServer
}
func (s *dtmServer) NewGid(ctx context.Context, in *emptypb.Empty) (*dtmgimp.DtmGidReply, error) {
return &dtmgimp.DtmGidReply{Gid: GenGid()}, nil
func (s *dtmServer) NewGid(ctx context.Context, in *emptypb.Empty) (*pb.DtmGidReply, error) {
return &pb.DtmGidReply{Gid: GenGid()}, nil
}
func (s *dtmServer) Submit(ctx context.Context, in *pb.DtmRequest) (*emptypb.Empty, error) {
r, err := svcSubmit(TransFromDtmRequest(in))
return &emptypb.Empty{}, dtmgimp.Result2Error(r, err)
r := svcSubmit(TransFromDtmRequest(ctx, in))
return &emptypb.Empty{}, dtmgrpc.DtmError2GrpcError(r)
}
func (s *dtmServer) Prepare(ctx context.Context, in *pb.DtmRequest) (*emptypb.Empty, error) {
r, err := svcPrepare(TransFromDtmRequest(in))
return &emptypb.Empty{}, dtmgimp.Result2Error(r, err)
r := svcPrepare(TransFromDtmRequest(ctx, in))
return &emptypb.Empty{}, dtmgrpc.DtmError2GrpcError(r)
}
func (s *dtmServer) Abort(ctx context.Context, in *pb.DtmRequest) (*emptypb.Empty, error) {
r, err := svcAbort(TransFromDtmRequest(in))
return &emptypb.Empty{}, dtmgimp.Result2Error(r, err)
r := svcAbort(TransFromDtmRequest(ctx, in))
return &emptypb.Empty{}, dtmgrpc.DtmError2GrpcError(r)
}
func (s *dtmServer) RegisterBranch(ctx context.Context, in *pb.DtmBranchRequest) (*emptypb.Empty, error) {
r, err := svcRegisterBranch(in.TransType, &TransBranch{
r := svcRegisterBranch(in.TransType, &TransBranch{
Gid: in.Gid,
BranchID: in.BranchID,
Status: dtmcli.StatusPrepared,
BinData: in.BusiPayload,
}, in.Data)
return &emptypb.Empty{}, dtmgimp.Result2Error(r, err)
return &emptypb.Empty{}, dtmgrpc.DtmError2GrpcError(r)
}

46
dtmsvr/api_http.go

@ -9,23 +9,23 @@ package dtmsvr
import (
"errors"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmutil"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
func addRoute(engine *gin.Engine) {
engine.GET("/api/dtmsvr/newGid", common.WrapHandler(newGid))
engine.POST("/api/dtmsvr/prepare", common.WrapHandler(prepare))
engine.POST("/api/dtmsvr/submit", common.WrapHandler(submit))
engine.POST("/api/dtmsvr/abort", common.WrapHandler(abort))
engine.POST("/api/dtmsvr/registerBranch", common.WrapHandler(registerBranch))
engine.POST("/api/dtmsvr/registerXaBranch", common.WrapHandler(registerBranch)) // compatible for old sdk
engine.POST("/api/dtmsvr/registerTccBranch", common.WrapHandler(registerBranch)) // compatible for old sdk
engine.GET("/api/dtmsvr/query", common.WrapHandler(query))
engine.GET("/api/dtmsvr/all", common.WrapHandler(all))
engine.GET("/api/dtmsvr/newGid", dtmutil.WrapHandler2(newGid))
engine.POST("/api/dtmsvr/prepare", dtmutil.WrapHandler2(prepare))
engine.POST("/api/dtmsvr/submit", dtmutil.WrapHandler2(submit))
engine.POST("/api/dtmsvr/abort", dtmutil.WrapHandler2(abort))
engine.POST("/api/dtmsvr/registerBranch", dtmutil.WrapHandler2(registerBranch))
engine.POST("/api/dtmsvr/registerXaBranch", dtmutil.WrapHandler2(registerBranch)) // compatible for old sdk
engine.POST("/api/dtmsvr/registerTccBranch", dtmutil.WrapHandler2(registerBranch)) // compatible for old sdk
engine.GET("/api/dtmsvr/query", dtmutil.WrapHandler2(query))
engine.GET("/api/dtmsvr/all", dtmutil.WrapHandler2(all))
// add prometheus exporter
h := promhttp.Handler()
@ -34,23 +34,23 @@ func addRoute(engine *gin.Engine) {
})
}
func newGid(c *gin.Context) (interface{}, error) {
return map[string]interface{}{"gid": GenGid(), "dtm_result": dtmcli.ResultSuccess}, nil
func newGid(c *gin.Context) interface{} {
return map[string]interface{}{"gid": GenGid(), "dtm_result": dtmcli.ResultSuccess}
}
func prepare(c *gin.Context) (interface{}, error) {
func prepare(c *gin.Context) interface{} {
return svcPrepare(TransFromContext(c))
}
func submit(c *gin.Context) (interface{}, error) {
func submit(c *gin.Context) interface{} {
return svcSubmit(TransFromContext(c))
}
func abort(c *gin.Context) (interface{}, error) {
func abort(c *gin.Context) interface{} {
return svcAbort(TransFromContext(c))
}
func registerBranch(c *gin.Context) (interface{}, error) {
func registerBranch(c *gin.Context) interface{} {
data := map[string]string{}
err := c.BindJSON(&data)
e2p(err)
@ -63,19 +63,19 @@ func registerBranch(c *gin.Context) (interface{}, error) {
return svcRegisterBranch(data["trans_type"], &branch, data)
}
func query(c *gin.Context) (interface{}, error) {
func query(c *gin.Context) interface{} {
gid := c.Query("gid")
if gid == "" {
return nil, errors.New("no gid specified")
return errors.New("no gid specified")
}
trans := GetStore().FindTransGlobalStore(gid)
branches := GetStore().FindBranches(gid)
return map[string]interface{}{"transaction": trans, "branches": branches}, nil
return map[string]interface{}{"transaction": trans, "branches": branches}
}
func all(c *gin.Context) (interface{}, error) {
func all(c *gin.Context) interface{} {
position := c.Query("position")
slimit := dtmimp.OrString(c.Query("limit"), "100")
globals := GetStore().ScanTransGlobalStores(&position, int64(dtmimp.MustAtoi(slimit)))
return map[string]interface{}{"transactions": globals, "next_position": position}, nil
return map[string]interface{}{"transactions": globals, "next_position": position}
}

107
dtmsvr/config/config.go

@ -0,0 +1,107 @@
package config
import (
"encoding/json"
"io/ioutil"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/logger"
"gopkg.in/yaml.v2"
)
const (
// DtmMetricsPort is metric port
DtmMetricsPort = 8889
// Mysql is mysql driver
Mysql = "mysql"
// Redis is redis driver
Redis = "redis"
// BoltDb is boltdb driver
BoltDb = "boltdb"
// Postgres is postgres driver
Postgres = "postgres"
)
// MicroService config type for micro service
type MicroService struct {
Driver string `yaml:"Driver" default:"default"`
Target string `yaml:"Target"`
EndPoint string `yaml:"EndPoint"`
}
// Log config customize log
type Log struct {
Level string `yaml:"Level" default:"info"`
Output string `yaml:"Output" default:"console"`
FileName string `yaml:"FileName" default:"/tmp/dtm.log"`
FileMaxSize int64 `yaml:"FileMaxSize" default:"10"`
FileMaxBackups int64 `yaml:"FileMaxBackups" default:"5"`
FileMaxAge int64 `yaml:"FileMaxAge" default:"30"`
FileCompress int64 `yaml:"FileCompress" default:"0"`
}
// Store defines storage relevant info
type Store struct {
Driver string `yaml:"Driver" default:"boltdb"`
Host string `yaml:"Host"`
Port int64 `yaml:"Port"`
User string `yaml:"User"`
Password string `yaml:"Password"`
MaxOpenConns int64 `yaml:"MaxOpenConns" default:"500"`
MaxIdleConns int64 `yaml:"MaxIdleConns" default:"500"`
ConnMaxLifeTime int64 `yaml:"ConnMaxLifeTime" default:"5"`
DataExpire int64 `yaml:"DataExpire" default:"604800"` // Trans data will expire in 7 days. only for redis/boltdb.
RedisPrefix string `yaml:"RedisPrefix" default:"{a}"` // Redis storage prefix. store data to only one slot in cluster
TransGlobalTable string `yaml:"TransGlobalTable" default:"dtm.trans_global"`
TransBranchOpTable string `yaml:"TransBranchOpTable" default:"dtm.trans_branch_op"`
}
// IsDB checks config driver is mysql or postgres
func (s *Store) IsDB() bool {
return s.Driver == dtmcli.DBTypeMysql || s.Driver == dtmcli.DBTypePostgres
}
// GetDBConf returns db conf info
func (s *Store) GetDBConf() dtmcli.DBConf {
return dtmcli.DBConf{
Driver: s.Driver,
Host: s.Host,
Port: s.Port,
User: s.User,
Password: s.Password,
}
}
type configType struct {
Store Store `yaml:"Store"`
TransCronInterval int64 `yaml:"TransCronInterval" default:"3"`
TimeoutToFail int64 `yaml:"TimeoutToFail" default:"35"`
RetryInterval int64 `yaml:"RetryInterval" default:"10"`
RequestTimeout int64 `yaml:"RequestTimeout" default:"3"`
HTTPPort int64 `yaml:"HttpPort" default:"36789"`
GrpcPort int64 `yaml:"GrpcPort" default:"36790"`
MicroService MicroService `yaml:"MicroService"`
UpdateBranchSync int64 `yaml:"UpdateBranchSync"`
UpdateBranchAsyncGoroutineNum int64 `yaml:"UpdateBranchAsyncGoroutineNum" default:"1"`
Log Log `yaml:"Log"`
}
// Config 配置
var Config = configType{}
// MustLoadConfig load config from env and file
func MustLoadConfig(confFile string) {
loadFromEnv("", &Config)
if confFile != "" {
cont, err := ioutil.ReadFile(confFile)
logger.FatalIfError(err)
err = yaml.UnmarshalStrict(cont, &Config)
logger.FatalIfError(err)
}
scont, err := json.MarshalIndent(&Config, "", " ")
logger.FatalIfError(err)
logger.Infof("config file: %s loaded config is: \n%s", confFile, scont)
err = checkConfig(&Config)
logger.FatalfIf(err != nil, `config error: '%v'.
please visit http://d.dtm.pub to see the config document.`, err)
}

84
dtmsvr/config/config_test.go

@ -0,0 +1,84 @@
package config
import (
"errors"
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadFromEnv(t *testing.T) {
assert.Equal(t, "MICRO_SERVICE_DRIVER", toUnderscoreUpper("MicroService_Driver"))
ms := MicroService{}
os.Setenv("T_DRIVER", "d1")
loadFromEnv("T", &ms)
assert.Equal(t, "d1", ms.Driver)
}
func TestLoadConfig(t *testing.T) {
MustLoadConfig("../../conf.sample.yml")
}
func TestCheckConfig(t *testing.T) {
conf := Config
conf.RetryInterval = 1
retryIntervalErr := checkConfig(&conf)
retryIntervalExpect := errors.New("RetryInterval should not be less than 10")
assert.Equal(t, retryIntervalErr, retryIntervalExpect)
conf.RetryInterval = 10
conf.TimeoutToFail = 5
timeoutToFailErr := checkConfig(&conf)
timeoutToFailExpect := errors.New("TimeoutToFail should not be less than RetryInterval")
assert.Equal(t, timeoutToFailErr, timeoutToFailExpect)
conf.TimeoutToFail = 20
driverErr := checkConfig(&conf)
assert.Equal(t, driverErr, nil)
conf.Store = Store{Driver: Mysql}
hostErr := checkConfig(&conf)
hostExpect := errors.New("Db host not valid ")
assert.Equal(t, hostErr, hostExpect)
conf.Store = Store{Driver: Mysql, Host: "127.0.0.1"}
portErr := checkConfig(&conf)
portExpect := errors.New("Db port not valid ")
assert.Equal(t, portErr, portExpect)
conf.Store = Store{Driver: Mysql, Host: "127.0.0.1", Port: 8686}
userErr := checkConfig(&conf)
userExpect := errors.New("Db user not valid ")
assert.Equal(t, userErr, userExpect)
conf.Store = Store{Driver: Redis, Host: "", Port: 8686}
assert.Equal(t, errors.New("Redis host not valid"), checkConfig(&conf))
conf.Store = Store{Driver: Redis, Host: "127.0.0.1", Port: 0}
assert.Equal(t, errors.New("Redis port not valid"), checkConfig(&conf))
}
func TestConfig(t *testing.T) {
testConfigStringField(&Config.Store.Driver, "", t)
testConfigStringField(&Config.Store.User, "", t)
testConfigIntField(&Config.RetryInterval, 9, t)
testConfigIntField(&Config.TimeoutToFail, 9, t)
}
func testConfigStringField(fd *string, val string, t *testing.T) {
old := *fd
*fd = val
str := checkConfig(&Config)
assert.NotEqual(t, "", str)
*fd = old
}
func testConfigIntField(fd *int64, val int64, t *testing.T) {
old := *fd
*fd = val
str := checkConfig(&Config)
assert.NotEqual(t, "", str)
*fd = old
}

43
common/config_utils.go → dtmsvr/config/config_utils.go

@ -1,13 +1,14 @@
package common
package config
import (
"errors"
"fmt"
"os"
"reflect"
"regexp"
"strings"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
)
func loadFromEnv(prefix string, conf interface{}) {
@ -48,8 +49,42 @@ func loadFromEnvInner(prefix string, conf reflect.Value, defaultValue string) {
func toUnderscoreUpper(key string) string {
key = strings.Trim(key, "_")
matchLastCap := regexp.MustCompile("([A-Z])([A-Z][a-z])")
s2 := matchLastCap.ReplaceAllString(key, "${1}_${2}")
matchFirstCap := regexp.MustCompile("([a-z])([A-Z]+)")
s2 := matchFirstCap.ReplaceAllString(key, "${1}_${2}")
// dtmimp.Logf("loading from env: %s", strings.ToUpper(s2))
s2 = matchFirstCap.ReplaceAllString(s2, "${1}_${2}")
// logger.Infof("loading from env: %s", strings.ToUpper(s2))
return strings.ToUpper(s2)
}
func checkConfig(conf *configType) error {
if conf.RetryInterval < 10 {
return errors.New("RetryInterval should not be less than 10")
}
if conf.TimeoutToFail < conf.RetryInterval {
return errors.New("TimeoutToFail should not be less than RetryInterval")
}
switch conf.Store.Driver {
case BoltDb:
return nil
case Mysql, Postgres:
if conf.Store.Host == "" {
return errors.New("Db host not valid ")
}
if conf.Store.Port == 0 {
return errors.New("Db port not valid ")
}
if conf.Store.User == "" {
return errors.New("Db user not valid ")
}
case Redis:
if conf.Store.Host == "" {
return errors.New("Redis host not valid")
}
if conf.Store.Port == 0 {
return errors.New("Redis port not valid")
}
}
return nil
}

20
dtmsvr/cron.go

@ -7,19 +7,22 @@
package dtmsvr
import (
"errors"
"fmt"
"math/rand"
"runtime/debug"
"time"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
)
// NowForwardDuration will be set in test, trans may be timeout
var NowForwardDuration time.Duration = time.Duration(0)
var NowForwardDuration = time.Duration(0)
// CronForwardDuration will be set in test. cron will fetch trans which expire in CronForwardDuration
var CronForwardDuration time.Duration = time.Duration(0)
var CronForwardDuration = time.Duration(0)
// CronTransOnce cron expired trans. use expireIn as expire time
func CronTransOnce() (gid string) {
@ -30,7 +33,9 @@ func CronTransOnce() (gid string) {
}
gid = trans.Gid
trans.WaitResult = true
trans.Process()
branches := GetStore().FindBranches(gid)
err := trans.Process(branches)
dtmimp.PanicIf(err != nil && !errors.Is(err, dtmcli.ErrFailure), err)
return
}
@ -49,12 +54,13 @@ func lockOneTrans(expireIn time.Duration) *TransGlobal {
if global == nil {
return nil
}
logger.Infof("cron job return a trans: %s", global.String())
return &TransGlobal{TransGlobalStore: *global}
}
func handlePanic(perr *error) {
if err := recover(); err != nil {
dtmimp.LogRedf("----recovered panic %v\n%s", err, string(debug.Stack()))
logger.Errorf("----recovered panic %v\n%s", err, string(debug.Stack()))
if perr != nil {
*perr = fmt.Errorf("dtm panic: %v", err)
}
@ -62,8 +68,8 @@ func handlePanic(perr *error) {
}
func sleepCronTime() {
normal := time.Duration((float64(config.TransCronInterval) - rand.Float64()) * float64(time.Second))
normal := time.Duration((float64(conf.TransCronInterval) - rand.Float64()) * float64(time.Second))
interval := dtmimp.If(CronForwardDuration > 0, 1*time.Millisecond, normal).(time.Duration)
dtmimp.Logf("sleeping for %v milli", interval/time.Microsecond)
logger.Debugf("sleeping for %v milli", interval/time.Microsecond)
time.Sleep(interval)
}

165
dtmsvr/storage/boltdb/boltdb.go

@ -1,48 +1,59 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package boltdb
import (
"fmt"
"strings"
"sync"
"time"
bolt "go.etcd.io/bbolt"
"gorm.io/gorm"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmutil"
)
var config = &common.Config
// Store implements storage.Store, and storage with boltdb
type Store struct {
boltDb *bolt.DB
type BoltdbStore struct {
dataExpire int64
retryInterval int64
}
var boltDb *bolt.DB = nil
var boltOnce sync.Once
// NewStore will return the boltdb implement
// TODO: change to options
func NewStore(dataExpire int64, retryInterval int64) *Store {
s := &Store{
dataExpire: dataExpire,
retryInterval: retryInterval,
}
func boltGet() *bolt.DB {
boltOnce.Do(func() {
db, err := bolt.Open("./dtm.bolt", 0666, &bolt.Options{Timeout: 1 * time.Second})
dtmimp.E2P(err)
db, err := bolt.Open("./dtm.bolt", 0666, &bolt.Options{Timeout: 1 * time.Second})
dtmimp.E2P(err)
// NOTE: we must ensure all buckets is exists before we use it
err = initializeBuckets(db)
dtmimp.E2P(err)
// NOTE: we must ensure all buckets is exists before we use it
err = initializeBuckets(db)
dtmimp.E2P(err)
// TODO:
// 1. refactor this code
// 2. make cleanup run period, to avoid the file growup when server long-running
err = cleanupExpiredData(
time.Duration(common.Config.Store.DataExpire)*time.Second,
db,
)
dtmimp.E2P(err)
// TODO:
// 1. refactor this code
// 2. make cleanup run period, to avoid the file growup when server long-running
err = cleanupExpiredData(
time.Duration(dataExpire)*time.Second,
db,
)
dtmimp.E2P(err)
boltDb = db
})
return boltDb
s.boltDb = db
return s
}
func initializeBuckets(db *bolt.DB) error {
@ -100,10 +111,10 @@ func cleanupGlobalWithGids(t *bolt.Tx, gids map[string]struct{}) {
return
}
dtmimp.Logf("Start to cleanup %d gids", len(gids))
logger.Debugf("Start to cleanup %d gids", len(gids))
for gid := range gids {
dtmimp.Logf("Start to delete gid: %s", gid)
bucket.Delete([]byte(gid))
logger.Debugf("Start to delete gid: %s", gid)
dtmimp.E2P(bucket.Delete([]byte(gid)))
}
}
@ -129,10 +140,10 @@ func cleanupBranchWithGids(t *bolt.Tx, gids map[string]struct{}) {
}
}
dtmimp.Logf("Start to cleanup %d branches", len(branchKeys))
logger.Debugf("Start to cleanup %d branches", len(branchKeys))
for _, key := range branchKeys {
dtmimp.Logf("Start to delete branch: %s", key)
bucket.Delete([]byte(key))
logger.Debugf("Start to delete branch: %s", key)
dtmimp.E2P(bucket.Delete([]byte(key)))
}
}
@ -155,10 +166,10 @@ func cleanupIndexWithGids(t *bolt.Tx, gids map[string]struct{}) {
}
}
dtmimp.Logf("Start to cleanup %d indexes", len(indexKeys))
logger.Debugf("Start to cleanup %d indexes", len(indexKeys))
for _, key := range indexKeys {
dtmimp.Logf("Start to delete index: %s", key)
bucket.Delete([]byte(key))
logger.Debugf("Start to delete index: %s", key)
dtmimp.E2P(bucket.Delete([]byte(key)))
}
}
@ -225,27 +236,35 @@ func tPutIndex(t *bolt.Tx, unix int64, gid string) {
dtmimp.E2P(err)
}
func (s *BoltdbStore) Ping() error {
// Ping execs ping cmd to boltdb
func (s *Store) Ping() error {
return nil
}
func (s *BoltdbStore) PopulateData(skipDrop bool) {
// PopulateData populates data to boltdb
func (s *Store) PopulateData(skipDrop bool) {
if !skipDrop {
err := boltGet().Update(func(t *bolt.Tx) error {
t.DeleteBucket(bucketIndex)
t.DeleteBucket(bucketBranches)
t.DeleteBucket(bucketGlobal)
t.CreateBucket(bucketIndex)
t.CreateBucket(bucketBranches)
t.CreateBucket(bucketGlobal)
err := s.boltDb.Update(func(t *bolt.Tx) error {
dtmimp.E2P(t.DeleteBucket(bucketIndex))
dtmimp.E2P(t.DeleteBucket(bucketBranches))
dtmimp.E2P(t.DeleteBucket(bucketGlobal))
_, err := t.CreateBucket(bucketIndex)
dtmimp.E2P(err)
_, err = t.CreateBucket(bucketBranches)
dtmimp.E2P(err)
_, err = t.CreateBucket(bucketGlobal)
dtmimp.E2P(err)
return nil
})
dtmimp.E2P(err)
logger.Infof("Reset all data for boltdb")
}
}
func (s *BoltdbStore) FindTransGlobalStore(gid string) (trans *storage.TransGlobalStore) {
err := boltGet().View(func(t *bolt.Tx) error {
// FindTransGlobalStore finds GlobalTrans data by gid
func (s *Store) FindTransGlobalStore(gid string) (trans *storage.TransGlobalStore) {
err := s.boltDb.View(func(t *bolt.Tx) error {
trans = tGetGlobal(t, gid)
return nil
})
@ -253,9 +272,10 @@ func (s *BoltdbStore) FindTransGlobalStore(gid string) (trans *storage.TransGlob
return
}
func (s *BoltdbStore) ScanTransGlobalStores(position *string, limit int64) []storage.TransGlobalStore {
// ScanTransGlobalStores lists GlobalTrans data
func (s *Store) ScanTransGlobalStores(position *string, limit int64) []storage.TransGlobalStore {
globals := []storage.TransGlobalStore{}
err := boltGet().View(func(t *bolt.Tx) error {
err := s.boltDb.View(func(t *bolt.Tx) error {
cursor := t.Bucket(bucketGlobal).Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
if string(k) == *position {
@ -279,9 +299,10 @@ func (s *BoltdbStore) ScanTransGlobalStores(position *string, limit int64) []sto
return globals
}
func (s *BoltdbStore) FindBranches(gid string) []storage.TransBranchStore {
var branches []storage.TransBranchStore = nil
err := boltGet().View(func(t *bolt.Tx) error {
// FindBranches finds Branch data by gid
func (s *Store) FindBranches(gid string) []storage.TransBranchStore {
var branches []storage.TransBranchStore
err := s.boltDb.View(func(t *bolt.Tx) error {
branches = tGetBranches(t, gid)
return nil
})
@ -289,12 +310,14 @@ func (s *BoltdbStore) FindBranches(gid string) []storage.TransBranchStore {
return branches
}
func (s *BoltdbStore) UpdateBranchesSql(branches []storage.TransBranchStore, updates []string) *gorm.DB {
return nil // not implemented
// UpdateBranches update branches info
func (s *Store) UpdateBranches(branches []storage.TransBranchStore, updates []string) (int, error) {
return 0, nil // not implemented
}
func (s *BoltdbStore) LockGlobalSaveBranches(gid string, status string, branches []storage.TransBranchStore, branchStart int) {
err := boltGet().Update(func(t *bolt.Tx) error {
// LockGlobalSaveBranches creates branches
func (s *Store) LockGlobalSaveBranches(gid string, status string, branches []storage.TransBranchStore, branchStart int) {
err := s.boltDb.Update(func(t *bolt.Tx) error {
g := tGetGlobal(t, gid)
if g == nil {
return storage.ErrNotFound
@ -308,8 +331,9 @@ func (s *BoltdbStore) LockGlobalSaveBranches(gid string, status string, branches
dtmimp.E2P(err)
}
func (s *BoltdbStore) MaySaveNewTrans(global *storage.TransGlobalStore, branches []storage.TransBranchStore) error {
return boltGet().Update(func(t *bolt.Tx) error {
// MaySaveNewTrans creates a new trans
func (s *Store) MaySaveNewTrans(global *storage.TransGlobalStore, branches []storage.TransBranchStore) error {
return s.boltDb.Update(func(t *bolt.Tx) error {
g := tGetGlobal(t, global.Gid)
if g != nil {
return storage.ErrUniqueConflict
@ -321,10 +345,11 @@ func (s *BoltdbStore) MaySaveNewTrans(global *storage.TransGlobalStore, branches
})
}
func (s *BoltdbStore) ChangeGlobalStatus(global *storage.TransGlobalStore, newStatus string, updates []string, finished bool) {
// ChangeGlobalStatus changes global trans status
func (s *Store) ChangeGlobalStatus(global *storage.TransGlobalStore, newStatus string, updates []string, finished bool) {
old := global.Status
global.Status = newStatus
err := boltGet().Update(func(t *bolt.Tx) error {
err := s.boltDb.Update(func(t *bolt.Tx) error {
g := tGetGlobal(t, global.Gid)
if g == nil || g.Status != old {
return storage.ErrNotFound
@ -338,12 +363,13 @@ func (s *BoltdbStore) ChangeGlobalStatus(global *storage.TransGlobalStore, newSt
dtmimp.E2P(err)
}
func (s *BoltdbStore) TouchCronTime(global *storage.TransGlobalStore, nextCronInterval int64) {
// TouchCronTime updates cronTime
func (s *Store) TouchCronTime(global *storage.TransGlobalStore, nextCronInterval int64) {
oldUnix := global.NextCronTime.Unix()
global.NextCronTime = common.GetNextTime(nextCronInterval)
global.UpdateTime = common.GetNextTime(0)
global.NextCronTime = dtmutil.GetNextTime(nextCronInterval)
global.UpdateTime = dtmutil.GetNextTime(0)
global.NextCronInterval = nextCronInterval
err := boltGet().Update(func(t *bolt.Tx) error {
err := s.boltDb.Update(func(t *bolt.Tx) error {
g := tGetGlobal(t, global.Gid)
if g == nil || g.Gid != global.Gid {
return storage.ErrNotFound
@ -356,13 +382,14 @@ func (s *BoltdbStore) TouchCronTime(global *storage.TransGlobalStore, nextCronIn
dtmimp.E2P(err)
}
func (s *BoltdbStore) LockOneGlobalTrans(expireIn time.Duration) *storage.TransGlobalStore {
var trans *storage.TransGlobalStore = nil
// LockOneGlobalTrans finds GlobalTrans
func (s *Store) LockOneGlobalTrans(expireIn time.Duration) *storage.TransGlobalStore {
var trans *storage.TransGlobalStore
min := fmt.Sprintf("%d", time.Now().Add(expireIn).Unix())
next := time.Now().Add(time.Duration(config.RetryInterval) * time.Second)
err := boltGet().Update(func(t *bolt.Tx) error {
next := time.Now().Add(time.Duration(s.retryInterval) * time.Second)
err := s.boltDb.Update(func(t *bolt.Tx) error {
cursor := t.Bucket(bucketIndex).Cursor()
for trans == nil {
for trans == nil || trans.Status == dtmcli.StatusSucceed || trans.Status == dtmcli.StatusFailed {
k, v := cursor.First()
if k == nil || string(k) > min {
return storage.ErrNotFound

10
dtmsvr/storage/boltdb/boltdb_test.go

@ -1,3 +1,9 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package boltdb
import (
@ -8,8 +14,8 @@ import (
. "github.com/onsi/gomega"
bolt "go.etcd.io/bbolt"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmsvr/storage"
)
func TestInitializeBuckets(t *testing.T) {

257
dtmsvr/storage/redis.go

@ -1,257 +0,0 @@
package storage
import (
"context"
"fmt"
"time"
"github.com/go-redis/redis/v8"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli/dtmimp"
"gorm.io/gorm"
)
var ctx context.Context = context.Background()
type RedisStore struct {
}
func (s *RedisStore) Ping() error {
_, err := redisGet().Ping(ctx).Result()
return err
}
func (s *RedisStore) PopulateData(skipDrop bool) {
_, err := redisGet().FlushAll(ctx).Result()
dtmimp.PanicIf(err != nil, err)
}
func (s *RedisStore) FindTransGlobalStore(gid string) *TransGlobalStore {
r, err := redisGet().Get(ctx, config.Store.RedisPrefix+"_g_"+gid).Result()
if err == redis.Nil {
return nil
}
dtmimp.E2P(err)
trans := &TransGlobalStore{}
dtmimp.MustUnmarshalString(r, trans)
return trans
}
func (s *RedisStore) ScanTransGlobalStores(position *string, limit int64) []TransGlobalStore {
lid := uint64(0)
if *position != "" {
lid = uint64(dtmimp.MustAtoi(*position))
}
keys, cursor, err := redisGet().Scan(ctx, lid, config.Store.RedisPrefix+"_g_*", limit).Result()
dtmimp.E2P(err)
globals := []TransGlobalStore{}
if len(keys) > 0 {
values, err := redisGet().MGet(ctx, keys...).Result()
dtmimp.E2P(err)
for _, v := range values {
global := TransGlobalStore{}
dtmimp.MustUnmarshalString(v.(string), &global)
globals = append(globals, global)
}
}
if cursor > 0 {
*position = fmt.Sprintf("%d", cursor)
} else {
*position = ""
}
return globals
}
func (s *RedisStore) FindBranches(gid string) []TransBranchStore {
sa, err := redisGet().LRange(ctx, config.Store.RedisPrefix+"_b_"+gid, 0, -1).Result()
dtmimp.E2P(err)
branches := make([]TransBranchStore, len(sa))
for k, v := range sa {
dtmimp.MustUnmarshalString(v, &branches[k])
}
return branches
}
func (s *RedisStore) UpdateBranchesSql(branches []TransBranchStore, updates []string) *gorm.DB {
return nil // not implemented
}
type argList struct {
Keys []string
List []interface{}
}
func newArgList() *argList {
a := &argList{}
return a.AppendRaw(config.Store.RedisPrefix).AppendObject(config.Store.DataExpire)
}
func (a *argList) AppendGid(gid string) *argList {
a.Keys = append(a.Keys, config.Store.RedisPrefix+"_g_"+gid)
a.Keys = append(a.Keys, config.Store.RedisPrefix+"_b_"+gid)
a.Keys = append(a.Keys, config.Store.RedisPrefix+"_u")
return a
}
func (a *argList) AppendRaw(v interface{}) *argList {
a.List = append(a.List, v)
return a
}
func (a *argList) AppendObject(v interface{}) *argList {
return a.AppendRaw(dtmimp.MustMarshalString(v))
}
func (a *argList) AppendBranches(branches []TransBranchStore) *argList {
for _, b := range branches {
a.AppendRaw(dtmimp.MustMarshalString(b))
}
return a
}
func handleRedisResult(ret interface{}, err error) (string, error) {
dtmimp.Logf("result is: '%v', err: '%v'", ret, err)
if err != nil && err != redis.Nil {
return "", err
}
s, _ := ret.(string)
err = map[string]error{
"NOT_FOUND": ErrNotFound,
"UNIQUE_CONFLICT": ErrUniqueConflict,
}[s]
return s, err
}
func callLua(a *argList, lua string) (string, error) {
dtmimp.Logf("calling lua. args: %v\nlua:%s", a, lua)
ret, err := redisGet().Eval(ctx, lua, a.Keys, a.List...).Result()
return handleRedisResult(ret, err)
}
func (s *RedisStore) MaySaveNewTrans(global *TransGlobalStore, branches []TransBranchStore) error {
a := newArgList().
AppendGid(global.Gid).
AppendObject(global).
AppendRaw(global.NextCronTime.Unix()).
AppendBranches(branches)
global.Steps = nil
global.Payloads = nil
_, err := callLua(a, `-- MaySaveNewTrans
local gs = cjson.decode(ARGV[3])
local g = redis.call('GET', KEYS[1])
if g ~= false then
return 'UNIQUE_CONFLICT'
end
redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2])
redis.call('ZADD', KEYS[3], ARGV[4], gs.gid)
for k = 5, table.getn(ARGV) do
redis.call('RPUSH', KEYS[2], ARGV[k])
end
redis.call('EXPIRE', KEYS[2], ARGV[2])
`)
return err
}
func (s *RedisStore) LockGlobalSaveBranches(gid string, status string, branches []TransBranchStore, branchStart int) {
args := newArgList().
AppendGid(gid).
AppendObject(&TransGlobalStore{Gid: gid, Status: status}).
AppendRaw(branchStart).
AppendBranches(branches)
_, err := callLua(args, `
local gs = cjson.decode(ARGV[3])
local g = redis.call('GET', KEYS[1])
if (g == false) then
return 'NOT_FOUND'
end
local js = cjson.decode(g)
if js.status ~= gs.status then
return 'NOT_FOUND'
end
local start = ARGV[4]
for k = 5, table.getn(ARGV) do
if start == "-1" then
redis.call('RPUSH', KEYS[2], ARGV[k])
else
redis.call('LSET', KEYS[2], start+k-5, ARGV[k])
end
end
redis.call('EXPIRE', KEYS[2], ARGV[2])
`)
dtmimp.E2P(err)
}
func (s *RedisStore) ChangeGlobalStatus(global *TransGlobalStore, newStatus string, updates []string, finished bool) {
old := global.Status
global.Status = newStatus
args := newArgList().AppendGid(global.Gid).AppendObject(global).AppendRaw(old).AppendRaw(finished)
_, err := callLua(args, `-- ChangeGlobalStatus
local gs = cjson.decode(ARGV[3])
local old = redis.call('GET', KEYS[1])
if old == false then
return 'NOT_FOUND'
end
local os = cjson.decode(old)
if os.status ~= ARGV[4] then
return 'NOT_FOUND'
end
redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2])
redis.log(redis.LOG_WARNING, 'finished: ', ARGV[5])
if ARGV[5] == '1' then
redis.call('ZREM', KEYS[3], gs.gid)
end
`)
dtmimp.E2P(err)
}
func (s *RedisStore) LockOneGlobalTrans(expireIn time.Duration) *TransGlobalStore {
expired := time.Now().Add(expireIn).Unix()
next := time.Now().Add(time.Duration(config.RetryInterval) * time.Second).Unix()
args := newArgList().AppendGid("").AppendRaw(expired).AppendRaw(next)
lua := `-- LocakOneGlobalTrans
local r = redis.call('ZRANGE', KEYS[3], 0, 0, 'WITHSCORES')
local gid = r[1]
if gid == nil then
return 'NOT_FOUND'
end
if tonumber(r[2]) > tonumber(ARGV[3]) then
return 'NOT_FOUND'
end
redis.call('ZADD', KEYS[3], ARGV[4], gid)
return gid
`
for {
r, err := callLua(args, lua)
if err == ErrNotFound {
return nil
}
dtmimp.E2P(err)
global := s.FindTransGlobalStore(r)
if global != nil {
return global
}
}
}
func (s *RedisStore) TouchCronTime(global *TransGlobalStore, nextCronInterval int64) {
global.NextCronTime = common.GetNextTime(nextCronInterval)
global.UpdateTime = common.GetNextTime(0)
global.NextCronInterval = nextCronInterval
args := newArgList().AppendGid(global.Gid).AppendObject(global).AppendRaw(global.NextCronTime.Unix())
_, err := callLua(args, `-- TouchCronTime
local g = cjson.decode(ARGV[3])
local old = redis.call('GET', KEYS[1])
if old == false then
return 'NOT_FOUND'
end
local os = cjson.decode(old)
if os.status ~= g.status then
return 'NOT_FOUND'
end
redis.call('ZADD', KEYS[3], ARGV[4], g.gid)
redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2])
`)
dtmimp.E2P(err)
}

300
dtmsvr/storage/redis/redis.go

@ -0,0 +1,300 @@
package redis
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/go-redis/redis/v8"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmsvr/config"
"github.com/dtm-labs/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmutil"
)
// TODO: optimize this, it's very strange to use pointer to dtmutil.Config
var conf = &config.Config
// TODO: optimize this, all function should have context as first parameter
var ctx = context.Background()
// Store is the storage with redis, all transaction information will bachend with redis
type Store struct {
}
// Ping execs ping cmd to redis
func (s *Store) Ping() error {
_, err := redisGet().Ping(ctx).Result()
return err
}
// PopulateData populates data to redis
func (s *Store) PopulateData(skipDrop bool) {
if !skipDrop {
_, err := redisGet().FlushAll(ctx).Result()
logger.Infof("call redis flushall. result: %v", err)
dtmimp.PanicIf(err != nil, err)
}
}
// FindTransGlobalStore finds GlobalTrans data by gid
func (s *Store) FindTransGlobalStore(gid string) *storage.TransGlobalStore {
logger.Debugf("calling FindTransGlobalStore: %s", gid)
r, err := redisGet().Get(ctx, conf.Store.RedisPrefix+"_g_"+gid).Result()
if err == redis.Nil {
return nil
}
dtmimp.E2P(err)
trans := &storage.TransGlobalStore{}
dtmimp.MustUnmarshalString(r, trans)
return trans
}
// ScanTransGlobalStores lists GlobalTrans data
func (s *Store) ScanTransGlobalStores(position *string, limit int64) []storage.TransGlobalStore {
logger.Debugf("calling ScanTransGlobalStores: %s %d", *position, limit)
lid := uint64(0)
if *position != "" {
lid = uint64(dtmimp.MustAtoi(*position))
}
keys, cursor, err := redisGet().Scan(ctx, lid, conf.Store.RedisPrefix+"_g_*", limit).Result()
dtmimp.E2P(err)
globals := []storage.TransGlobalStore{}
if len(keys) > 0 {
values, err := redisGet().MGet(ctx, keys...).Result()
dtmimp.E2P(err)
for _, v := range values {
global := storage.TransGlobalStore{}
dtmimp.MustUnmarshalString(v.(string), &global)
globals = append(globals, global)
}
}
if cursor > 0 {
*position = fmt.Sprintf("%d", cursor)
} else {
*position = ""
}
return globals
}
// FindBranches finds Branch data by gid
func (s *Store) FindBranches(gid string) []storage.TransBranchStore {
logger.Debugf("calling FindBranches: %s", gid)
sa, err := redisGet().LRange(ctx, conf.Store.RedisPrefix+"_b_"+gid, 0, -1).Result()
dtmimp.E2P(err)
branches := make([]storage.TransBranchStore, len(sa))
for k, v := range sa {
dtmimp.MustUnmarshalString(v, &branches[k])
}
return branches
}
// UpdateBranches updates branches info
func (s *Store) UpdateBranches(branches []storage.TransBranchStore, updates []string) (int, error) {
return 0, nil // not implemented
}
type argList struct {
Keys []string
List []interface{}
}
func newArgList() *argList {
a := &argList{}
return a.AppendRaw(conf.Store.RedisPrefix).AppendObject(conf.Store.DataExpire)
}
func (a *argList) AppendGid(gid string) *argList {
a.Keys = append(a.Keys, conf.Store.RedisPrefix+"_g_"+gid)
a.Keys = append(a.Keys, conf.Store.RedisPrefix+"_b_"+gid)
a.Keys = append(a.Keys, conf.Store.RedisPrefix+"_u")
a.Keys = append(a.Keys, conf.Store.RedisPrefix+"_s_"+gid)
return a
}
func (a *argList) AppendRaw(v interface{}) *argList {
a.List = append(a.List, v)
return a
}
func (a *argList) AppendObject(v interface{}) *argList {
return a.AppendRaw(dtmimp.MustMarshalString(v))
}
func (a *argList) AppendBranches(branches []storage.TransBranchStore) *argList {
for _, b := range branches {
a.AppendRaw(dtmimp.MustMarshalString(b))
}
return a
}
func handleRedisResult(ret interface{}, err error) (string, error) {
logger.Debugf("result is: '%v', err: '%v'", ret, err)
if err != nil && err != redis.Nil {
return "", err
}
s, _ := ret.(string)
err = map[string]error{
"NOT_FOUND": storage.ErrNotFound,
"UNIQUE_CONFLICT": storage.ErrUniqueConflict,
}[s]
return s, err
}
func callLua(a *argList, lua string) (string, error) {
logger.Debugf("calling lua. args: %v\nlua:%s", a, lua)
ret, err := redisGet().Eval(ctx, lua, a.Keys, a.List...).Result()
return handleRedisResult(ret, err)
}
// MaySaveNewTrans creates a new trans
func (s *Store) MaySaveNewTrans(global *storage.TransGlobalStore, branches []storage.TransBranchStore) error {
a := newArgList().
AppendGid(global.Gid).
AppendObject(global).
AppendRaw(global.NextCronTime.Unix()).
AppendRaw(global.Gid).
AppendRaw(global.Status).
AppendBranches(branches)
global.Steps = nil
global.Payloads = nil
_, err := callLua(a, `-- MaySaveNewTrans
local g = redis.call('GET', KEYS[1])
if g ~= false then
return 'UNIQUE_CONFLICT'
end
redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2])
redis.call('SET', KEYS[4], ARGV[6], 'EX', ARGV[2])
redis.call('ZADD', KEYS[3], ARGV[4], ARGV[5])
for k = 7, table.getn(ARGV) do
redis.call('RPUSH', KEYS[2], ARGV[k])
end
redis.call('EXPIRE', KEYS[2], ARGV[2])
`)
return err
}
// LockGlobalSaveBranches creates branches
func (s *Store) LockGlobalSaveBranches(gid string, status string, branches []storage.TransBranchStore, branchStart int) {
args := newArgList().
AppendGid(gid).
AppendRaw(status).
AppendRaw(branchStart).
AppendBranches(branches)
_, err := callLua(args, `-- LockGlobalSaveBranches
local old = redis.call('GET', KEYS[4])
if old ~= ARGV[3] then
return 'NOT_FOUND'
end
local start = ARGV[4]
for k = 5, table.getn(ARGV) do
if start == "-1" then
redis.call('RPUSH', KEYS[2], ARGV[k])
else
redis.call('LSET', KEYS[2], start+k-5, ARGV[k])
end
end
redis.call('EXPIRE', KEYS[2], ARGV[2])
`)
dtmimp.E2P(err)
}
// ChangeGlobalStatus changes global trans status
func (s *Store) ChangeGlobalStatus(global *storage.TransGlobalStore, newStatus string, updates []string, finished bool) {
old := global.Status
global.Status = newStatus
args := newArgList().
AppendGid(global.Gid).
AppendObject(global).
AppendRaw(old).
AppendRaw(finished).
AppendRaw(global.Gid).
AppendRaw(newStatus)
_, err := callLua(args, `-- ChangeGlobalStatus
local old = redis.call('GET', KEYS[4])
if old ~= ARGV[4] then
return 'NOT_FOUND'
end
redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2])
redis.call('SET', KEYS[4], ARGV[7], 'EX', ARGV[2])
if ARGV[5] == '1' then
redis.call('ZREM', KEYS[3], ARGV[6])
end
`)
dtmimp.E2P(err)
}
// LockOneGlobalTrans finds GlobalTrans
func (s *Store) LockOneGlobalTrans(expireIn time.Duration) *storage.TransGlobalStore {
expired := time.Now().Add(expireIn).Unix()
next := time.Now().Add(time.Duration(conf.RetryInterval) * time.Second).Unix()
args := newArgList().AppendGid("").AppendRaw(expired).AppendRaw(next)
lua := `-- LockOneGlobalTrans
local r = redis.call('ZRANGE', KEYS[3], 0, 0, 'WITHSCORES')
local gid = r[1]
if gid == nil then
return 'NOT_FOUND'
end
if tonumber(r[2]) > tonumber(ARGV[3]) then
return 'NOT_FOUND'
end
redis.call('ZADD', KEYS[3], ARGV[4], gid)
return gid
`
for {
r, err := callLua(args, lua)
if errors.Is(err, storage.ErrNotFound) {
return nil
}
dtmimp.E2P(err)
global := s.FindTransGlobalStore(r)
if global != nil {
return global
}
}
}
// TouchCronTime updates cronTime
func (s *Store) TouchCronTime(global *storage.TransGlobalStore, nextCronInterval int64) {
global.NextCronTime = dtmutil.GetNextTime(nextCronInterval)
global.UpdateTime = dtmutil.GetNextTime(0)
global.NextCronInterval = nextCronInterval
args := newArgList().
AppendGid(global.Gid).
AppendObject(global).
AppendRaw(global.NextCronTime.Unix()).
AppendRaw(global.Status).
AppendRaw(global.Gid)
_, err := callLua(args, `-- TouchCronTime
local old = redis.call('GET', KEYS[4])
if old ~= ARGV[5] then
return 'NOT_FOUND'
end
redis.call('ZADD', KEYS[3], ARGV[4], ARGV[6])
redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2])
`)
dtmimp.E2P(err)
}
var (
rdb *redis.Client
once sync.Once
)
func redisGet() *redis.Client {
once.Do(func() {
logger.Debugf("connecting to redis: %v", conf.Store)
rdb = redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%d", conf.Store.Host, conf.Store.Port),
Username: conf.Store.User,
Password: conf.Store.Password,
})
})
return rdb
}

25
dtmsvr/storage/registry/factory.go

@ -0,0 +1,25 @@
package registry
import (
"sync"
"github.com/dtm-labs/dtm/dtmsvr/storage"
)
// SingletonFactory is the factory to build store in SINGLETON pattern.
type SingletonFactory struct {
once sync.Once
store storage.Store
creatorFunction func() storage.Store
}
// GetStorage implement the StorageFactory.GetStorage
func (f *SingletonFactory) GetStorage() storage.Store {
f.once.Do(func() {
f.store = f.creatorFunction()
})
return f.store
}

45
dtmsvr/storage/registry/registry.go

@ -3,22 +3,47 @@ package registry
import (
"time"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmsvr/storage"
"github.com/yedf/dtm/dtmsvr/storage/boltdb"
"github.com/dtm-labs/dtm/dtmsvr/config"
"github.com/dtm-labs/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmsvr/storage/boltdb"
"github.com/dtm-labs/dtm/dtmsvr/storage/redis"
"github.com/dtm-labs/dtm/dtmsvr/storage/sql"
)
var config = &common.Config
var conf = &config.Config
var stores map[string]storage.Store = map[string]storage.Store{
"redis": &storage.RedisStore{},
"mysql": &storage.SqlStore{},
"postgres": &storage.SqlStore{},
"boltdb": &boltdb.BoltdbStore{},
// StorageFactory is factory to get storage instance.
type StorageFactory interface {
// GetStorage will return the Storage instance.
GetStorage() storage.Store
}
var storeFactorys = map[string]StorageFactory{
"boltdb": &SingletonFactory{
creatorFunction: func() storage.Store {
return boltdb.NewStore(conf.Store.DataExpire, conf.RetryInterval)
},
},
"redis": &SingletonFactory{
creatorFunction: func() storage.Store {
return &redis.Store{}
},
},
"mysql": &SingletonFactory{
creatorFunction: func() storage.Store {
return &sql.Store{}
},
},
"postgres": &SingletonFactory{
creatorFunction: func() storage.Store {
return &sql.Store{}
},
},
}
// GetStore returns storage.Store
func GetStore() storage.Store {
return stores[config.Store.Driver]
return storeFactorys[conf.Store.Driver].GetStorage()
}
// WaitStoreUp wait for db to go up

140
dtmsvr/storage/sql.go

@ -1,140 +0,0 @@
package storage
import (
"fmt"
"math"
"time"
"github.com/google/uuid"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli/dtmimp"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type SqlStore struct {
}
func (s *SqlStore) Ping() error {
db, err := dtmimp.StandaloneDB(config.Store.GetDBConf())
dtmimp.E2P(err)
_, err = db.Exec("select 1")
return err
}
func (s *SqlStore) PopulateData(skipDrop bool) {
file := fmt.Sprintf("%s/dtmsvr.storage.%s.sql", common.GetSqlDir(), config.Store.Driver)
common.RunSQLScript(config.Store.GetDBConf(), file, skipDrop)
}
func (s *SqlStore) FindTransGlobalStore(gid string) *TransGlobalStore {
trans := &TransGlobalStore{}
dbr := dbGet().Model(trans).Where("gid=?", gid).First(trans)
if dbr.Error == gorm.ErrRecordNotFound {
return nil
}
dtmimp.E2P(dbr.Error)
return trans
}
func (s *SqlStore) ScanTransGlobalStores(position *string, limit int64) []TransGlobalStore {
globals := []TransGlobalStore{}
lid := math.MaxInt64
if *position != "" {
lid = dtmimp.MustAtoi(*position)
}
dbr := dbGet().Must().Where("id < ?", lid).Order("id desc").Limit(int(limit)).Find(&globals)
if dbr.RowsAffected < limit {
*position = ""
} else {
*position = fmt.Sprintf("%d", globals[len(globals)-1].ID)
}
return globals
}
func (s *SqlStore) FindBranches(gid string) []TransBranchStore {
branches := []TransBranchStore{}
dbGet().Must().Where("gid=?", gid).Order("id asc").Find(&branches)
return branches
}
func (s *SqlStore) UpdateBranchesSql(branches []TransBranchStore, updates []string) *gorm.DB {
return dbGet().Clauses(clause.OnConflict{
OnConstraint: "trans_branch_op_pkey",
DoUpdates: clause.AssignmentColumns(updates),
}).Create(branches)
}
func (s *SqlStore) LockGlobalSaveBranches(gid string, status string, branches []TransBranchStore, branchStart int) {
err := dbGet().Transaction(func(tx *gorm.DB) error {
g := &TransGlobalStore{}
dbr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Model(g).Where("gid=? and status=?", gid, status).First(g)
if dbr.Error == nil {
dbr = tx.Save(branches)
}
return wrapError(dbr.Error)
})
dtmimp.E2P(err)
}
func (s *SqlStore) MaySaveNewTrans(global *TransGlobalStore, branches []TransBranchStore) error {
return dbGet().Transaction(func(db1 *gorm.DB) error {
db := &common.DB{DB: db1}
dbr := db.Must().Clauses(clause.OnConflict{
DoNothing: true,
}).Create(global)
if dbr.RowsAffected <= 0 { // 如果这个不是新事务,返回错误
return ErrUniqueConflict
}
if len(branches) > 0 {
db.Must().Clauses(clause.OnConflict{
DoNothing: true,
}).Create(&branches)
}
return nil
})
}
func (s *SqlStore) ChangeGlobalStatus(global *TransGlobalStore, newStatus string, updates []string, finished bool) {
old := global.Status
global.Status = newStatus
dbr := dbGet().Must().Model(global).Where("status=? and gid=?", old, global.Gid).Select(updates).Updates(global)
if dbr.RowsAffected == 0 {
dtmimp.E2P(ErrNotFound)
}
}
func (s *SqlStore) TouchCronTime(global *TransGlobalStore, nextCronInterval int64) {
global.NextCronTime = common.GetNextTime(nextCronInterval)
global.UpdateTime = common.GetNextTime(0)
global.NextCronInterval = nextCronInterval
dbGet().Must().Model(global).Where("status=? and gid=?", global.Status, global.Gid).
Select([]string{"next_cron_time", "update_time", "next_cron_interval"}).Updates(global)
}
func (s *SqlStore) LockOneGlobalTrans(expireIn time.Duration) *TransGlobalStore {
db := dbGet()
getTime := func(second int) string {
return map[string]string{
"mysql": fmt.Sprintf("date_add(now(), interval %d second)", second),
"postgres": fmt.Sprintf("current_timestamp + interval '%d second'", second),
}[config.Store.Driver]
}
expire := int(expireIn / time.Second)
whereTime := fmt.Sprintf("next_cron_time < %s", getTime(expire))
owner := uuid.NewString()
global := &TransGlobalStore{}
dbr := db.Must().Model(global).
Where(whereTime + "and status in ('prepared', 'aborting', 'submitted')").
Limit(1).
Select([]string{"owner", "next_cron_time"}).
Updates(&TransGlobalStore{
Owner: owner,
NextCronTime: common.GetNextTime(common.Config.RetryInterval),
})
if dbr.RowsAffected == 0 {
return nil
}
dbr = db.Must().Where("owner=?", owner).First(global)
return global
}

178
dtmsvr/storage/sql/sql.go

@ -0,0 +1,178 @@
package sql
import (
"fmt"
"math"
"time"
"github.com/lithammer/shortuuid/v3"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmsvr/config"
"github.com/dtm-labs/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmutil"
)
var conf = &config.Config
// Store implements storage.Store, and storage with db
type Store struct {
}
// Ping execs ping cmd to db
func (s *Store) Ping() error {
db, err := dtmimp.StandaloneDB(conf.Store.GetDBConf())
dtmimp.E2P(err)
_, err = db.Exec("select 1")
return err
}
// PopulateData populates data to db
func (s *Store) PopulateData(skipDrop bool) {
file := fmt.Sprintf("%s/dtmsvr.storage.%s.sql", dtmutil.GetSQLDir(), conf.Store.Driver)
dtmutil.RunSQLScript(conf.Store.GetDBConf(), file, skipDrop)
}
// FindTransGlobalStore finds GlobalTrans data by gid
func (s *Store) FindTransGlobalStore(gid string) *storage.TransGlobalStore {
trans := &storage.TransGlobalStore{}
dbr := dbGet().Model(trans).Where("gid=?", gid).First(trans)
if dbr.Error == gorm.ErrRecordNotFound {
return nil
}
dtmimp.E2P(dbr.Error)
return trans
}
// ScanTransGlobalStores lists GlobalTrans data
func (s *Store) ScanTransGlobalStores(position *string, limit int64) []storage.TransGlobalStore {
globals := []storage.TransGlobalStore{}
lid := math.MaxInt64
if *position != "" {
lid = dtmimp.MustAtoi(*position)
}
dbr := dbGet().Must().Where("id < ?", lid).Order("id desc").Limit(int(limit)).Find(&globals)
if dbr.RowsAffected < limit {
*position = ""
} else {
*position = fmt.Sprintf("%d", globals[len(globals)-1].ID)
}
return globals
}
// FindBranches finds Branch data by gid
func (s *Store) FindBranches(gid string) []storage.TransBranchStore {
branches := []storage.TransBranchStore{}
dbGet().Must().Where("gid=?", gid).Order("id asc").Find(&branches)
return branches
}
// UpdateBranches update branches info
func (s *Store) UpdateBranches(branches []storage.TransBranchStore, updates []string) (int, error) {
db := dbGet().Clauses(clause.OnConflict{
OnConstraint: "trans_branch_op_pkey",
DoUpdates: clause.AssignmentColumns(updates),
}).Create(branches)
return int(db.RowsAffected), db.Error
}
// LockGlobalSaveBranches creates branches
func (s *Store) LockGlobalSaveBranches(gid string, status string, branches []storage.TransBranchStore, branchStart int) {
err := dbGet().Transaction(func(tx *gorm.DB) error {
g := &storage.TransGlobalStore{}
dbr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Model(g).Where("gid=? and status=?", gid, status).First(g)
if dbr.Error == nil {
dbr = tx.Save(branches)
}
return wrapError(dbr.Error)
})
dtmimp.E2P(err)
}
// MaySaveNewTrans creates a new trans
func (s *Store) MaySaveNewTrans(global *storage.TransGlobalStore, branches []storage.TransBranchStore) error {
return dbGet().Transaction(func(db1 *gorm.DB) error {
db := &dtmutil.DB{DB: db1}
dbr := db.Must().Clauses(clause.OnConflict{
DoNothing: true,
}).Create(global)
if dbr.RowsAffected <= 0 { // 如果这个不是新事务,返回错误
return storage.ErrUniqueConflict
}
if len(branches) > 0 {
db.Must().Clauses(clause.OnConflict{
DoNothing: true,
}).Create(&branches)
}
return nil
})
}
// ChangeGlobalStatus changes global trans status
func (s *Store) ChangeGlobalStatus(global *storage.TransGlobalStore, newStatus string, updates []string, finished bool) {
old := global.Status
global.Status = newStatus
dbr := dbGet().Must().Model(global).Where("status=? and gid=?", old, global.Gid).Select(updates).Updates(global)
if dbr.RowsAffected == 0 {
dtmimp.E2P(storage.ErrNotFound)
}
}
// TouchCronTime updates cronTime
func (s *Store) TouchCronTime(global *storage.TransGlobalStore, nextCronInterval int64) {
global.NextCronTime = dtmutil.GetNextTime(nextCronInterval)
global.UpdateTime = dtmutil.GetNextTime(0)
global.NextCronInterval = nextCronInterval
dbGet().Must().Model(global).Where("status=? and gid=?", global.Status, global.Gid).
Select([]string{"next_cron_time", "update_time", "next_cron_interval"}).Updates(global)
}
// LockOneGlobalTrans finds GlobalTrans
func (s *Store) LockOneGlobalTrans(expireIn time.Duration) *storage.TransGlobalStore {
db := dbGet()
getTime := func(second int) string {
return map[string]string{
"mysql": fmt.Sprintf("date_add(now(), interval %d second)", second),
"postgres": fmt.Sprintf("current_timestamp + interval '%d second'", second),
}[conf.Store.Driver]
}
expire := int(expireIn / time.Second)
whereTime := fmt.Sprintf("next_cron_time < %s", getTime(expire))
owner := shortuuid.New()
global := &storage.TransGlobalStore{}
dbr := db.Must().Model(global).
Where(whereTime + "and status in ('prepared', 'aborting', 'submitted')").
Limit(1).
Select([]string{"owner", "next_cron_time"}).
Updates(&storage.TransGlobalStore{
Owner: owner,
NextCronTime: dtmutil.GetNextTime(conf.RetryInterval),
})
if dbr.RowsAffected == 0 {
return nil
}
db.Must().Where("owner=?", owner).First(global)
return global
}
// SetDBConn sets db conn pool
func SetDBConn(db *gorm.DB) {
sqldb, _ := db.DB()
sqldb.SetMaxOpenConns(int(conf.Store.MaxOpenConns))
sqldb.SetMaxIdleConns(int(conf.Store.MaxIdleConns))
sqldb.SetConnMaxLifetime(time.Duration(conf.Store.ConnMaxLifeTime) * time.Minute)
}
func dbGet() *dtmutil.DB {
return dtmutil.DbGet(conf.Store.GetDBConf(), SetDBConn)
}
func wrapError(err error) error {
if err == gorm.ErrRecordNotFound {
return storage.ErrNotFound
}
dtmimp.E2P(err)
return err
}

24
dtmsvr/storage/store.go

@ -1,35 +1,33 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package storage
import (
"errors"
"time"
"github.com/go-redis/redis/v8"
"github.com/yedf/dtm/dtmcli/dtmimp"
"gorm.io/gorm"
)
// ErrNotFound defines the query item is not found in storage implement.
var ErrNotFound = errors.New("storage: NotFound")
// ErrUniqueConflict defines the item is conflict with unique key in storage implement.
var ErrUniqueConflict = errors.New("storage: UniqueKeyConflict")
// Store defines storage relevant interface
type Store interface {
Ping() error
PopulateData(skipDrop bool)
FindTransGlobalStore(gid string) *TransGlobalStore
ScanTransGlobalStores(position *string, limit int64) []TransGlobalStore
FindBranches(gid string) []TransBranchStore
UpdateBranchesSql(branches []TransBranchStore, updates []string) *gorm.DB
UpdateBranches(branches []TransBranchStore, updates []string) (int, error)
LockGlobalSaveBranches(gid string, status string, branches []TransBranchStore, branchStart int)
MaySaveNewTrans(global *TransGlobalStore, branches []TransBranchStore) error
ChangeGlobalStatus(global *TransGlobalStore, newStatus string, updates []string, finished bool)
TouchCronTime(global *TransGlobalStore, nextCronInterval int64)
LockOneGlobalTrans(expireIn time.Duration) *TransGlobalStore
}
func wrapError(err error) error {
if err == gorm.ErrRecordNotFound || err == redis.Nil {
return ErrNotFound
}
dtmimp.E2P(err)
return err
}

35
dtmsvr/storage/trans.go

@ -3,12 +3,20 @@ package storage
import (
"time"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmsvr/config"
"github.com/dtm-labs/dtm/dtmutil"
)
// TransGlobalExt defines Header info
type TransGlobalExt struct {
Headers map[string]string `json:"headers,omitempty" gorm:"-"`
}
// TransGlobalStore defines GlobalStore storage info
type TransGlobalStore struct {
common.ModelBase
dtmutil.ModelBase
Gid string `json:"gid,omitempty"`
TransType string `json:"trans_type,omitempty"`
Steps []map[string]string `json:"steps,omitempty" gorm:"-"`
@ -17,7 +25,6 @@ type TransGlobalStore struct {
Status string `json:"status,omitempty"`
QueryPrepared string `json:"query_prepared,omitempty"`
Protocol string `json:"protocol,omitempty"`
CommitTime *time.Time `json:"commit_time,omitempty"`
FinishTime *time.Time `json:"finish_time,omitempty"`
RollbackTime *time.Time `json:"rollback_time,omitempty"`
Options string `json:"options,omitempty"`
@ -25,17 +32,23 @@ type TransGlobalStore struct {
NextCronInterval int64 `json:"next_cron_interval,omitempty"`
NextCronTime *time.Time `json:"next_cron_time,omitempty"`
Owner string `json:"owner,omitempty"`
Ext TransGlobalExt `json:"-" gorm:"-"`
ExtData string `json:"ext_data,omitempty"` // storage of ext. a db field to store many values. like Options
dtmcli.TransOptions
}
// TableName TableName
func (*TransGlobalStore) TableName() string {
return "dtm.trans_global"
func (g *TransGlobalStore) TableName() string {
return config.Config.Store.TransGlobalTable
}
func (g *TransGlobalStore) String() string {
return dtmimp.MustMarshalString(g)
}
// TransBranchStore branch transaction
type TransBranchStore struct {
common.ModelBase
dtmutil.ModelBase
Gid string `json:"gid,omitempty"`
URL string `json:"url,omitempty"`
BinData []byte
@ -47,6 +60,10 @@ type TransBranchStore struct {
}
// TableName TableName
func (*TransBranchStore) TableName() string {
return "dtm.trans_branch_op"
func (b *TransBranchStore) TableName() string {
return config.Config.Store.TransBranchOpTable
}
func (b *TransBranchStore) String() string {
return dtmimp.MustMarshalString(*b)
}

16
dtmsvr/storage/utils.go

@ -1,16 +0,0 @@
package storage
import (
"github.com/go-redis/redis/v8"
"github.com/yedf/dtm/common"
)
var config = &common.Config
func dbGet() *common.DB {
return common.DbGet(config.Store.GetDBConf())
}
func redisGet() *redis.Client {
return common.RedisGet()
}

73
dtmsvr/svr.go

@ -7,46 +7,64 @@
package dtmsvr
import (
"context"
"fmt"
"net"
"time"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmgrpc"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
"github.com/dtm-labs/dtm/dtmutil"
"github.com/dtm-labs/dtmdriver"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/yedf/dtmdriver"
"google.golang.org/grpc"
)
// StartSvr StartSvr
func StartSvr() {
dtmimp.Logf("start dtmsvr")
app := common.GetGinApp()
logger.Infof("start dtmsvr")
dtmcli.GetRestyClient().SetTimeout(time.Duration(conf.RequestTimeout) * time.Second)
dtmgrpc.AddUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
ctx2, cancel := context.WithTimeout(ctx, time.Duration(conf.RequestTimeout)*time.Second)
defer cancel()
return invoker(ctx2, method, req, reply, cc, opts...)
})
app := dtmutil.GetGinApp()
app = httpMetrics(app)
addRoute(app)
dtmimp.Logf("dtmsvr listen at: %d", config.HttpPort)
go app.Run(fmt.Sprintf(":%d", config.HttpPort))
logger.Infof("dtmsvr listen at: %d", conf.HTTPPort)
go func() {
err := app.Run(fmt.Sprintf(":%d", conf.HTTPPort))
if err != nil {
logger.Errorf("start server err: %v", err)
}
}()
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.GrpcPort))
dtmimp.FatalIfError(err)
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", conf.GrpcPort))
logger.FatalIfError(err)
s := grpc.NewServer(
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
grpc.UnaryServerInterceptor(grpcMetrics), grpc.UnaryServerInterceptor(dtmgimp.GrpcServerLog)),
))
dtmgimp.RegisterDtmServer(s, &dtmServer{})
dtmimp.Logf("grpc listening at %v", lis.Addr())
dtmgpb.RegisterDtmServer(s, &dtmServer{})
logger.Infof("grpc listening at %v", lis.Addr())
go func() {
err := s.Serve(lis)
dtmimp.FatalIfError(err)
logger.FatalIfError(err)
}()
go updateBranchAsync()
for i := 0; i < int(conf.UpdateBranchAsyncGoroutineNum); i++ {
go updateBranchAsync()
}
time.Sleep(100 * time.Millisecond)
err = dtmdriver.Use(config.MicroService.Driver)
dtmimp.FatalIfError(err)
err = dtmdriver.GetDriver().RegisterGrpcService(config.MicroService.Target, config.MicroService.EndPoint)
dtmimp.FatalIfError(err)
err = dtmdriver.Use(conf.MicroService.Driver)
logger.FatalIfError(err)
err = dtmdriver.GetDriver().RegisterGrpcService(conf.MicroService.Target, conf.MicroService.EndPoint)
logger.FatalIfError(err)
}
// PopulateDB setup mysql data
@ -59,8 +77,8 @@ var UpdateBranchAsyncInterval = 200 * time.Millisecond
var updateBranchAsyncChan chan branchStatus = make(chan branchStatus, 1000)
func updateBranchAsync() {
for { // flush branches every second
defer common.RecoverPanic(nil)
flushBranchs := func() {
defer dtmutil.RecoverPanic(nil)
updates := []TransBranch{}
started := time.Now()
checkInterval := 20 * time.Millisecond
@ -68,7 +86,8 @@ func updateBranchAsync() {
select {
case updateBranch := <-updateBranchAsyncChan:
updates = append(updates, TransBranch{
ModelBase: common.ModelBase{ID: updateBranch.id},
ModelBase: dtmutil.ModelBase{ID: updateBranch.id},
Gid: updateBranch.gid,
Status: updateBranch.status,
FinishTime: updateBranch.finishTime,
})
@ -76,15 +95,19 @@ func updateBranchAsync() {
}
}
for len(updates) > 0 {
dbr := GetStore().UpdateBranchesSql(updates, []string{"status", "finish_time", "update_time"})
rowAffected, err := GetStore().UpdateBranches(updates, []string{"status", "finish_time", "update_time"})
dtmimp.Logf("flushed %d branch status to db. affected: %d", len(updates), dbr.RowsAffected)
if dbr.Error != nil {
dtmimp.LogRedf("async update branch status error: %v", dbr.Error)
if err != nil {
logger.Errorf("async update branch status error: %v", err)
time.Sleep(1 * time.Second)
} else {
logger.Infof("flushed %d branch status to db. affected: %d", len(updates), rowAffected)
updates = []TransBranch{}
}
}
}
for { // flush branches every 200ms
flushBranchs()
}
}

7
dtmsvr/svr_imports.go

@ -1,7 +0,0 @@
package dtmsvr
import (
_ "github.com/ychensha/dtmdriver-polaris"
_ "github.com/yedf/dtmdriver-gozero"
_ "github.com/yedf/dtmdriver-protocol1"
)

44
dtmsvr/trans_class.go

@ -7,13 +7,16 @@
package dtmsvr
import (
"context"
"time"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgpb"
"github.com/dtm-labs/dtm/dtmsvr/storage"
"github.com/gin-gonic/gin"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/yedf/dtm/dtmsvr/storage"
)
// TransGlobal global transaction
@ -57,7 +60,7 @@ func TransFromContext(c *gin.Context) *TransGlobal {
e2p(err)
m := TransGlobal{}
dtmimp.MustUnmarshal(b, &m)
dtmimp.Logf("creating trans in prepare")
logger.Debugf("creating trans in prepare")
// Payloads will be store in BinPayloads, Payloads is only used to Unmarshal
for _, p := range m.Payloads {
m.BinPayloads = append(m.BinPayloads, []byte(p))
@ -68,12 +71,22 @@ func TransFromContext(c *gin.Context) *TransGlobal {
}
}
m.Protocol = "http"
m.Ext.Headers = map[string]string{}
if len(m.PassthroughHeaders) > 0 {
for _, h := range m.PassthroughHeaders {
v := c.GetHeader(h)
if v != "" {
m.Ext.Headers[h] = v
}
}
}
return &m
}
// TransFromDtmRequest TransFromContext
func TransFromDtmRequest(c *dtmgimp.DtmRequest) *TransGlobal {
o := &dtmgimp.DtmTransOptions{}
func TransFromDtmRequest(ctx context.Context, c *dtmgpb.DtmRequest) *TransGlobal {
o := &dtmgpb.DtmTransOptions{}
if c.TransOptions != nil {
o = c.TransOptions
}
@ -84,13 +97,24 @@ func TransFromDtmRequest(c *dtmgimp.DtmRequest) *TransGlobal {
Protocol: "grpc",
BinPayloads: c.BinPayloads,
TransOptions: dtmcli.TransOptions{
WaitResult: o.WaitResult,
TimeoutToFail: o.TimeoutToFail,
RetryInterval: o.RetryInterval,
WaitResult: o.WaitResult,
TimeoutToFail: o.TimeoutToFail,
RetryInterval: o.RetryInterval,
PassthroughHeaders: o.PassthroughHeaders,
BranchHeaders: o.BranchHeaders,
},
}}
if c.Steps != "" {
dtmimp.MustUnmarshalString(c.Steps, &r.Steps)
}
if len(o.PassthroughHeaders) > 0 {
r.Ext.Headers = map[string]string{}
for _, h := range o.PassthroughHeaders {
v := dtmgimp.GetMetaFromContext(ctx, h)
if v != "" {
r.Ext.Headers[h] = v
}
}
}
return &r
}

68
dtmsvr/trans_process.go

@ -7,63 +7,75 @@
package dtmsvr
import (
"fmt"
"time"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmutil"
)
// Process process global transaction once
func (t *TransGlobal) Process() map[string]interface{} {
r := t.process()
transactionMetrics(t, r["dtm_result"] == dtmcli.ResultSuccess)
func (t *TransGlobal) Process(branches []TransBranch) error {
r := t.process(branches)
transactionMetrics(t, r == nil)
return r
}
func (t *TransGlobal) process() map[string]interface{} {
func (t *TransGlobal) process(branches []TransBranch) error {
if t.Options != "" {
dtmimp.MustUnmarshalString(t.Options, &t.TransOptions)
}
if t.ExtData != "" {
dtmimp.MustUnmarshalString(t.ExtData, &t.Ext)
}
if !t.WaitResult {
go t.processInner()
return dtmcli.MapSuccess
go func() {
err := t.processInner(branches)
if err != nil {
logger.Errorf("processInner err: %v", err)
}
}()
return nil
}
submitting := t.Status == dtmcli.StatusSubmitted
err := t.processInner()
err := t.processInner(branches)
if err != nil {
return map[string]interface{}{"dtm_result": dtmcli.ResultFailure, "message": err.Error()}
return err
}
if submitting && t.Status != dtmcli.StatusSucceed {
return map[string]interface{}{"dtm_result": dtmcli.ResultFailure, "message": "trans failed by user"}
return fmt.Errorf("wait result not return success: %w", dtmcli.ErrFailure)
}
return dtmcli.MapSuccess
return nil
}
func (t *TransGlobal) processInner() (rerr error) {
func (t *TransGlobal) processInner(branches []TransBranch) (rerr error) {
defer handlePanic(&rerr)
defer func() {
if rerr != nil {
dtmimp.LogRedf("processInner got error: %s", rerr.Error())
if rerr != nil && rerr != dtmcli.ErrOngoing {
logger.Errorf("processInner got error: %s", rerr.Error())
}
if TransProcessedTestChan != nil {
dtmimp.Logf("processed: %s", t.Gid)
logger.Debugf("processed: %s", t.Gid)
TransProcessedTestChan <- t.Gid
dtmimp.Logf("notified: %s", t.Gid)
logger.Debugf("notified: %s", t.Gid)
}
}()
dtmimp.Logf("processing: %s status: %s", t.Gid, t.Status)
branches := GetStore().FindBranches(t.Gid)
logger.Debugf("processing: %s status: %s", t.Gid, t.Status)
t.lastTouched = time.Now()
rerr = t.getProcessor().ProcessOnce(branches)
return
}
func (t *TransGlobal) saveNew() error {
branches := t.getProcessor().GenBranches()
func (t *TransGlobal) saveNew() ([]TransBranch, error) {
t.NextCronInterval = t.getNextCronInterval(cronReset)
t.NextCronTime = common.GetNextTime(t.NextCronInterval)
t.NextCronTime = dtmutil.GetNextTime(t.NextCronInterval)
t.ExtData = dtmimp.MustMarshalString(t.Ext)
if t.ExtData == "{}" {
t.ExtData = ""
}
t.Options = dtmimp.MustMarshalString(t.TransOptions)
if t.Options == "{}" {
t.Options = ""
@ -71,5 +83,13 @@ func (t *TransGlobal) saveNew() error {
now := time.Now()
t.CreateTime = &now
t.UpdateTime = &now
return GetStore().MaySaveNewTrans(&t.TransGlobalStore, branches)
branches := t.getProcessor().GenBranches()
for i := range branches {
branches[i].CreateTime = &now
branches[i].UpdateTime = &now
}
err := GetStore().MaySaveNewTrans(&t.TransGlobalStore, branches)
logger.Infof("MaySaveNewTrans result: %v, global: %v branches: %v",
err, t.TransGlobalStore.String(), dtmimp.MustMarshalString(branches))
return branches, err
}

68
dtmsvr/trans_status.go

@ -7,21 +7,25 @@
package dtmsvr
import (
"errors"
"fmt"
"strings"
"time"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc/dtmgimp"
"github.com/yedf/dtmdriver"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
"github.com/dtm-labs/dtm/dtmgrpc/dtmgimp"
"github.com/dtm-labs/dtmdriver"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
func (t *TransGlobal) touchCronTime(ctype cronType) {
t.lastTouched = time.Now()
GetStore().TouchCronTime(&t.TransGlobalStore, t.getNextCronInterval(ctype))
logger.Infof("TouchCronTime for: %s", t.TransGlobalStore.String())
}
func (t *TransGlobal) changeStatus(status string) {
@ -36,6 +40,7 @@ func (t *TransGlobal) changeStatus(status string) {
}
t.UpdateTime = &now
GetStore().ChangeGlobalStatus(&t.TransGlobalStore, status, updates, status == dtmcli.StatusSucceed || status == dtmcli.StatusFailed)
logger.Infof("ChangeGlobalStatus to %s ok for %s", status, t.TransGlobalStore.String())
t.Status = status
}
@ -44,17 +49,19 @@ func (t *TransGlobal) changeBranchStatus(b *TransBranch, status string, branchPo
b.Status = status
b.FinishTime = &now
b.UpdateTime = &now
if config.Store.Driver != dtmimp.DBTypeMysql && config.Store.Driver != dtmimp.DBTypePostgres || config.UpdateBranchSync > 0 || t.updateBranchSync {
if conf.Store.Driver != dtmimp.DBTypeMysql && conf.Store.Driver != dtmimp.DBTypePostgres || conf.UpdateBranchSync > 0 || t.updateBranchSync {
GetStore().LockGlobalSaveBranches(t.Gid, t.Status, []TransBranch{*b}, branchPos)
logger.Infof("LockGlobalSaveBranches ok: gid: %s old status: %s branches: %s",
b.Gid, dtmcli.StatusPrepared, b.String())
} else { // 为了性能优化,把branch的status更新异步化
updateBranchAsyncChan <- branchStatus{id: b.ID, status: status, finishTime: &now}
updateBranchAsyncChan <- branchStatus{id: b.ID, gid: t.Gid, status: status, finishTime: &now}
}
}
func (t *TransGlobal) isTimeout() bool {
timeout := t.TimeoutToFail
if t.TimeoutToFail == 0 && t.TransType != "saga" {
timeout = config.TimeoutToFail
timeout = conf.TimeoutToFail
}
if timeout == 0 {
return false
@ -66,28 +73,36 @@ func (t *TransGlobal) needProcess() bool {
return t.Status == dtmcli.StatusSubmitted || t.Status == dtmcli.StatusAborting || t.Status == dtmcli.StatusPrepared && t.isTimeout()
}
func (t *TransGlobal) getURLResult(url string, branchID, op string, branchPayload []byte) (string, error) {
func (t *TransGlobal) getURLResult(url string, branchID, op string, branchPayload []byte) error {
if url == "" { // empty url is success
return nil
}
if t.Protocol == "grpc" {
dtmimp.PanicIf(strings.HasPrefix(url, "http"), fmt.Errorf("bad url for grpc: %s", url))
server, method, err := dtmdriver.GetDriver().ParseServerMethod(url)
if err != nil {
return "", err
return err
}
conn := dtmgimp.MustGetGrpcConn(server, true)
ctx := dtmgimp.TransInfo2Ctx(t.Gid, t.TransType, branchID, op, "")
kvs := dtmgimp.Map2Kvs(t.Ext.Headers)
kvs = append(kvs, dtmgimp.Map2Kvs(t.BranchHeaders)...)
ctx = metadata.AppendToOutgoingContext(ctx, kvs...)
err = conn.Invoke(ctx, method, branchPayload, &[]byte{})
if err == nil {
return dtmcli.ResultSuccess, nil
return nil
}
st, ok := status.FromError(err)
if ok && st.Code() == codes.Aborted {
// version lower then v1.10, will specify Ongoing in code Aborted
if st.Message() == dtmcli.ResultOngoing {
return dtmcli.ResultOngoing, nil
} else if st.Message() == dtmcli.ResultFailure {
return dtmcli.ResultFailure, nil
return dtmcli.ErrOngoing
}
return dtmcli.ErrFailure
} else if ok && st.Code() == codes.FailedPrecondition {
return dtmcli.ErrOngoing
}
return "", err
return err
}
dtmimp.PanicIf(!strings.HasPrefix(url, "http"), fmt.Errorf("bad url for http: %s", url))
resp, err := dtmimp.RestyClient.R().SetBody(string(branchPayload)).
@ -98,26 +113,25 @@ func (t *TransGlobal) getURLResult(url string, branchID, op string, branchPayloa
"op": op,
}).
SetHeader("Content-type", "application/json").
SetHeaders(t.Ext.Headers).
SetHeaders(t.TransOptions.BranchHeaders).
Execute(dtmimp.If(branchPayload != nil || t.TransType == "xa", "POST", "GET").(string), url)
if err != nil {
return "", err
return err
}
return resp.String(), nil
return dtmimp.RespAsErrorCompatible(resp)
}
func (t *TransGlobal) getBranchResult(branch *TransBranch) (string, error) {
body, err := t.getURLResult(branch.URL, branch.BranchID, branch.Op, branch.BinData)
if err != nil {
return "", err
}
if strings.Contains(body, dtmcli.ResultSuccess) {
err := t.getURLResult(branch.URL, branch.BranchID, branch.Op, branch.BinData)
if err == nil {
return dtmcli.StatusSucceed, nil
} else if strings.HasSuffix(t.TransType, "saga") && branch.Op == dtmcli.BranchAction && strings.Contains(body, dtmcli.ResultFailure) {
} else if t.TransType == "saga" && branch.Op == dtmcli.BranchAction && errors.Is(err, dtmcli.ErrFailure) {
return dtmcli.StatusFailed, nil
} else if strings.Contains(body, dtmcli.ResultOngoing) {
return "", dtmimp.ErrOngoing
} else if errors.Is(err, dtmcli.ErrOngoing) {
return "", dtmcli.ErrOngoing
}
return "", fmt.Errorf("http result should contains SUCCESS|FAILURE|ONGOING. grpc error should return nil|Aborted with message(FAILURE|ONGOING). \nrefer to: https://dtm.pub/summary/arch.html#http\nunkown result will be retried: %s", body)
return "", fmt.Errorf("http/grpc result should be specified as in:\nhttps://dtm.pub/summary/arch.html#http\nunkown result will be retried: %s", err)
}
func (t *TransGlobal) execBranch(branch *TransBranch, branchPos int) error {
@ -128,7 +142,7 @@ func (t *TransGlobal) execBranch(branch *TransBranch, branchPos int) error {
branchMetrics(t, branch, status == dtmcli.StatusSucceed)
// if time pass 1500ms and NextCronInterval is not default, then reset NextCronInterval
if err == nil && time.Since(t.lastTouched)+NowForwardDuration >= 1500*time.Millisecond ||
t.NextCronInterval > config.RetryInterval && t.NextCronInterval > t.RetryInterval {
t.NextCronInterval > conf.RetryInterval && t.NextCronInterval > t.RetryInterval {
t.touchCronTime(cronReset)
} else if err == dtmimp.ErrOngoing {
t.touchCronTime(cronKeep)
@ -146,6 +160,6 @@ func (t *TransGlobal) getNextCronInterval(ctype cronType) int64 {
} else if t.RetryInterval != 0 {
return t.RetryInterval
} else {
return config.RetryInterval
return conf.RetryInterval
}
}

16
dtmsvr/trans_type_msg.go

@ -7,11 +7,11 @@
package dtmsvr
import (
"errors"
"fmt"
"strings"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/logger"
)
type transMsgProcessor struct {
@ -42,15 +42,15 @@ func (t *TransGlobal) mayQueryPrepared() {
if !t.needProcess() || t.Status == dtmcli.StatusSubmitted {
return
}
body, err := t.getURLResult(t.QueryPrepared, "", "", nil)
if strings.Contains(body, dtmcli.ResultSuccess) {
err := t.getURLResult(t.QueryPrepared, "00", "msg", nil)
if err == nil {
t.changeStatus(dtmcli.StatusSubmitted)
} else if strings.Contains(body, dtmcli.ResultFailure) {
} else if errors.Is(err, dtmcli.ErrFailure) {
t.changeStatus(dtmcli.StatusFailed)
} else if strings.Contains(body, dtmcli.ResultOngoing) {
} else if errors.Is(err, dtmcli.ErrOngoing) {
t.touchCronTime(cronReset)
} else {
dtmimp.LogRedf("getting result failed for %s. error: %s", t.QueryPrepared, err.Error())
logger.Errorf("getting result failed for %s. error: %v", t.QueryPrepared, err)
t.touchCronTime(cronBackoff)
}
}

120
dtmsvr/trans_type_saga.go

@ -7,11 +7,13 @@
package dtmsvr
import (
"errors"
"fmt"
"time"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
)
type transSagaProcessor struct {
@ -19,7 +21,9 @@ type transSagaProcessor struct {
}
func init() {
registorProcessorCreator("saga", func(trans *TransGlobal) transProcessor { return &transSagaProcessor{TransGlobal: trans} })
registorProcessorCreator("saga", func(trans *TransGlobal) transProcessor {
return &transSagaProcessor{TransGlobal: trans}
})
}
func (t *transSagaProcessor) GenBranches() []TransBranch {
@ -43,6 +47,7 @@ func (t *transSagaProcessor) GenBranches() []TransBranch {
type cSagaCustom struct {
Orders map[int][]int `json:"orders"`
Concurrent bool `json:"concurrent"`
cOrders map[int][]int
}
type branchResult struct {
@ -54,15 +59,20 @@ type branchResult struct {
func (t *transSagaProcessor) ProcessOnce(branches []TransBranch) error {
// when saga tasks is fetched, it always need to process
dtmimp.Logf("status: %s timeout: %t", t.Status, t.isTimeout())
logger.Debugf("status: %s timeout: %t", t.Status, t.isTimeout())
if t.Status == dtmcli.StatusSubmitted && t.isTimeout() {
t.changeStatus(dtmcli.StatusAborting)
}
n := len(branches)
csc := cSagaCustom{Orders: map[int][]int{}}
csc := cSagaCustom{Orders: map[int][]int{}, cOrders: map[int][]int{}}
if t.CustomData != "" {
dtmimp.MustUnmarshalString(t.CustomData, &csc)
for k, v := range csc.Orders {
for _, b := range v {
csc.cOrders[b] = append(csc.cOrders[b], k)
}
}
}
if csc.Concurrent || t.TimeoutToFail > 0 { // when saga is not normal, update branch sync
t.updateBranchSync = true
@ -79,22 +89,41 @@ func (t *transSagaProcessor) ProcessOnce(branches []TransBranch) error {
rsAFailed++
}
}
branchResults[i] = branchResult{status: branches[i].Status, op: branches[i].Op}
branchResults[i] = branchResult{index: i, status: branches[i].Status, op: branches[i].Op}
}
isPreconditionsSucceed := func(current int) bool {
shouldRun := func(current int) bool {
// if !csc.Concurrent,then check the branch in previous step is succeed
if !csc.Concurrent && current >= 2 && branches[current-2].Status != dtmcli.StatusSucceed {
if !csc.Concurrent && current >= 2 && branchResults[current-2].status != dtmcli.StatusSucceed {
return false
}
// if csc.concurrent, then check the Orders. origin one step correspond to 2 step in dtmsvr
for _, pre := range csc.Orders[current/2] {
if branches[pre*2+1].Status != dtmcli.StatusSucceed {
if branchResults[pre*2+1].status != dtmcli.StatusSucceed {
return false
}
}
return true
}
shouldRollback := func(current int) bool {
rollbacked := func(i int) bool {
// current compensate op rollbacked or related action still prepared
return branchResults[i].status == dtmcli.StatusSucceed || branchResults[i+1].status == dtmcli.StatusPrepared
}
if rollbacked(current) {
return false
}
// if !csc.Concurrent,then check the branch in next step is rollbacked
if !csc.Concurrent && current < n-2 && !rollbacked(current+2) {
return false
}
// if csc.concurrent, then check the cOrders. origin one step correspond to 2 step in dtmsvr
for _, next := range csc.cOrders[current/2] {
if !rollbacked(2 * next) {
return false
}
}
return true
}
resultChan := make(chan branchResult, n)
asyncExecBranch := func(i int) {
var err error
@ -103,21 +132,32 @@ func (t *transSagaProcessor) ProcessOnce(branches []TransBranch) error {
err = dtmimp.AsError(x)
}
resultChan <- branchResult{index: i, status: branches[i].Status, op: branches[i].Op}
if err != nil {
dtmimp.LogRedf("exec branch error: %v", err)
if err != nil && !errors.Is(err, dtmcli.ErrOngoing) {
logger.Errorf("exec branch error: %v", err)
}
}()
err = t.execBranch(&branches[i], i)
}
pickToRunActions := func() []int {
toRun := []int{}
for current := 0; current < n; current++ {
for current := 1; current < n; current += 2 {
br := &branchResults[current]
if !br.started && br.status == dtmcli.StatusPrepared && shouldRun(current) {
toRun = append(toRun, current)
}
}
logger.Debugf("toRun picked for action is: %v branchResults: %v compensate orders: %v", toRun, branchResults, csc.cOrders)
return toRun
}
pickToRunCompensates := func() []int {
toRun := []int{}
for current := n - 2; current >= 0; current -= 2 {
br := &branchResults[current]
if br.op == dtmcli.BranchAction && !br.started && isPreconditionsSucceed(current) && br.status == dtmcli.StatusPrepared {
if !br.started && br.status == dtmcli.StatusPrepared && shouldRollback(current) {
toRun = append(toRun, current)
}
}
dtmimp.Logf("toRun picked for action is: %v", toRun)
logger.Debugf("toRun picked for compensate is: %v branchResults: %v compensate orders: %v", toRun, branchResults, csc.cOrders)
return toRun
}
runBranches := func(toRun []int) {
@ -129,18 +169,6 @@ func (t *transSagaProcessor) ProcessOnce(branches []TransBranch) error {
go asyncExecBranch(b)
}
}
pickAndRunCompensates := func(toRunActions []int) {
for _, b := range toRunActions {
// these branches may have run. so flag them to status succeed, then run the corresponding compensate
branchResults[b].status = dtmcli.StatusSucceed
}
for i, b := range branchResults {
if b.op == dtmcli.BranchCompensate && b.status != dtmcli.StatusSucceed && branchResults[i+1].status != dtmcli.StatusPrepared {
rsCToStart++
go asyncExecBranch(i)
}
}
}
waitDoneOnce := func() {
select {
case r := <-resultChan:
@ -159,13 +187,30 @@ func (t *transSagaProcessor) ProcessOnce(branches []TransBranch) error {
rsCSucceed++
}
}
dtmimp.Logf("branch done: %v", r)
logger.Debugf("branch done: %v", r)
case <-time.After(time.Duration(time.Second * 3)):
dtmimp.Logf("wait once for done")
logger.Debugf("wait once for done")
}
}
for t.Status == dtmcli.StatusSubmitted && !t.isTimeout() && rsAFailed == 0 && rsADone != rsAToStart {
prepareToCompensate := func() {
_ = pickToRunActions() // flag started
for i := 1; i < len(branchResults); i += 2 {
// these branches may have run. so flag them to status succeed, then run the corresponding
// compensate
if branchResults[i].started && branchResults[i].status == dtmcli.StatusPrepared {
branchResults[i].status = dtmcli.StatusSucceed
}
}
for i, b := range branchResults {
if b.op == dtmcli.BranchCompensate && b.status != dtmcli.StatusSucceed &&
branchResults[i+1].status != dtmcli.StatusPrepared {
rsCToStart++
}
}
logger.Debugf("rsCToStart: %d branchResults: %v", rsCToStart, branchResults)
}
timeLimit := time.Now().Add(time.Duration(conf.RequestTimeout+2) * time.Second)
for time.Now().Before(timeLimit) && t.Status == dtmcli.StatusSubmitted && !t.isTimeout() && rsAFailed == 0 {
toRun := pickToRunActions()
runBranches(toRun)
if rsADone == rsAStarted { // no branch is running, so break
@ -181,11 +226,16 @@ func (t *transSagaProcessor) ProcessOnce(branches []TransBranch) error {
t.changeStatus(dtmcli.StatusAborting)
}
if t.Status == dtmcli.StatusAborting {
toRun := pickToRunActions()
pickAndRunCompensates(toRun)
for rsCDone != rsCToStart {
waitDoneOnce()
prepareToCompensate()
}
for time.Now().Before(timeLimit) && t.Status == dtmcli.StatusAborting {
toRun := pickToRunCompensates()
runBranches(toRun)
if rsCDone == rsCToStart { // no branch is running, so break
break
}
logger.Debugf("rsCDone: %d rsCToStart: %d", rsCDone, rsCToStart)
waitDoneOnce()
}
if t.Status == dtmcli.StatusAborting && rsCToStart == rsCSucceed {
t.changeStatus(dtmcli.StatusFailed)

7
dtmsvr/trans_type_tcc.go

@ -7,8 +7,9 @@
package dtmsvr
import (
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
)
type transTccProcessor struct {
@ -33,7 +34,7 @@ func (t *transTccProcessor) ProcessOnce(branches []TransBranch) error {
op := dtmimp.If(t.Status == dtmcli.StatusSubmitted, dtmcli.BranchConfirm, dtmcli.BranchCancel).(string)
for current := len(branches) - 1; current >= 0; current-- {
if branches[current].Op == op && branches[current].Status == dtmcli.StatusPrepared {
dtmimp.Logf("branch info: current: %d ID: %d", current, branches[current].ID)
logger.Debugf("branch info: current: %d ID: %d", current, branches[current].ID)
err := t.execBranch(&branches[current], current)
if err != nil {
return err

4
dtmsvr/trans_type_xa.go

@ -7,8 +7,8 @@
package dtmsvr
import (
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
)
type transXaProcessor struct {

22
dtmsvr/utils.go

@ -10,40 +10,42 @@ import (
"fmt"
"time"
"github.com/google/uuid"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmsvr/storage"
"github.com/yedf/dtm/dtmsvr/storage/registry"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmsvr/config"
"github.com/dtm-labs/dtm/dtmsvr/storage"
"github.com/dtm-labs/dtm/dtmsvr/storage/registry"
"github.com/lithammer/shortuuid/v3"
)
type branchStatus struct {
id uint64
gid string
status string
finishTime *time.Time
}
var p2e = dtmimp.P2E
var e2p = dtmimp.E2P
var config = &common.Config
var conf = &config.Config
// GetStore returns storage.Store
func GetStore() storage.Store {
return registry.GetStore()
}
// TransProcessedTestChan only for test usage. when transaction processed once, write gid to this chan
var TransProcessedTestChan chan string = nil
var TransProcessedTestChan chan string
// GenGid generate gid, use uuid
func GenGid() string {
return uuid.NewString()
return shortuuid.New()
}
// GetTransGlobal construct trans from db
func GetTransGlobal(gid string) *TransGlobal {
trans := GetStore().FindTransGlobalStore(gid)
//nolint:staticcheck
dtmimp.PanicIf(trans == nil, fmt.Errorf("no TransGlobal with gid: %s found", gid))
//nolint:staticcheck
return &TransGlobal{TransGlobalStore: *trans}
}

4
dtmsvr/utils_test.go

@ -22,6 +22,6 @@ func TestSetNextCron(t *testing.T) {
tg.RetryInterval = 15
assert.Equal(t, int64(15), tg.getNextCronInterval(cronReset))
tg.RetryInterval = 0
assert.Equal(t, config.RetryInterval, tg.getNextCronInterval(cronReset))
assert.Equal(t, config.RetryInterval*2, tg.getNextCronInterval(cronBackoff))
assert.Equal(t, conf.RetryInterval, tg.getNextCronInterval(cronReset))
assert.Equal(t, conf.RetryInterval*2, tg.getNextCronInterval(cronBackoff))
}

14
dtmutil/consts.go

@ -0,0 +1,14 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package dtmutil
const (
// DefaultHTTPServer default url for http server. used by test and examples
DefaultHTTPServer = "http://localhost:36789/api/dtmsvr"
// DefaultGrpcServer default url for grpc server. used by test and examples
DefaultGrpcServer = "localhost:36790"
)

32
common/db.go → dtmutil/db.go

@ -1,4 +1,4 @@
package common
package dtmutil
import (
"database/sql"
@ -7,10 +7,11 @@ import (
"sync"
"time"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
_ "github.com/go-sql-driver/mysql" // register mysql driver
_ "github.com/lib/pq" // register postgres driver
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/gorm"
@ -27,7 +28,7 @@ func getGormDialetor(driver string, dsn string) gorm.Dialector {
if driver == dtmcli.DBTypePostgres {
return postgres.Open(dsn)
}
dtmimp.PanicIf(driver != dtmcli.DBTypeMysql, fmt.Errorf("unkown driver: %s", driver))
dtmimp.PanicIf(driver != dtmcli.DBTypeMysql, fmt.Errorf("unknown driver: %s", driver))
return mysql.Open(dsn)
}
@ -65,7 +66,7 @@ func (op *tracePlugin) Initialize(db *gorm.DB) (err error) {
after := func(db *gorm.DB) {
_ts, _ := db.InstanceGet("ivy.startTime")
sql := db.Dialector.Explain(db.Statement.SQL.String(), db.Statement.Vars...)
dtmimp.Logf("used: %d ms affected: %d sql is: %s", time.Since(_ts.(time.Time)).Milliseconds(), db.RowsAffected, sql)
logger.Debugf("used: %d ms affected: %d sql is: %s", time.Since(_ts.(time.Time)).Milliseconds(), db.RowsAffected, sql)
if v, ok := db.InstanceGet("ivy.must"); ok && v.(bool) {
if db.Error != nil && db.Error != gorm.ErrRecordNotFound {
panic(db.Error)
@ -76,7 +77,7 @@ func (op *tracePlugin) Initialize(db *gorm.DB) (err error) {
beforeName := "cb_before"
afterName := "cb_after"
dtmimp.Logf("installing db plugin: %s", op.Name())
logger.Debugf("installing db plugin: %s", op.Name())
// 开始前
_ = db.Callback().Create().Before("gorm:before_create").Register(beforeName, before)
_ = db.Callback().Query().Before("gorm:query").Register(beforeName, before)
@ -95,27 +96,22 @@ func (op *tracePlugin) Initialize(db *gorm.DB) (err error) {
return
}
// SetDBConn set db connection conf
func SetDBConn(db *DB) {
sqldb, _ := db.DB.DB()
sqldb.SetMaxOpenConns(int(Config.Store.MaxOpenConns))
sqldb.SetMaxIdleConns(int(Config.Store.MaxIdleConns))
sqldb.SetConnMaxLifetime(time.Duration(Config.Store.ConnMaxLifeTime) * time.Minute)
}
// DbGet get db connection for specified conf
func DbGet(conf dtmcli.DBConf) *DB {
func DbGet(conf dtmcli.DBConf, ops ...func(*gorm.DB)) *DB {
dsn := dtmimp.GetDsn(conf)
db, ok := dbs.Load(dsn)
if !ok {
dtmimp.Logf("connecting %s", strings.Replace(dsn, conf.Passwrod, "****", 1))
logger.Debugf("connecting %s", strings.Replace(dsn, conf.Password, "****", 1))
db1, err := gorm.Open(getGormDialetor(conf.Driver, dsn), &gorm.Config{
SkipDefaultTransaction: true,
})
dtmimp.E2P(err)
db1.Use(&tracePlugin{})
err = db1.Use(&tracePlugin{})
dtmimp.E2P(err)
db = &DB{DB: db1}
SetDBConn(db.(*DB))
for _, op := range ops {
op(db1)
}
dbs.Store(dsn, db)
}
return db.(*DB)

157
dtmutil/utils.go

@ -0,0 +1,157 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package dtmutil
import (
"bytes"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/go-resty/resty/v2"
"github.com/dtm-labs/dtm/dtmcli"
"github.com/dtm-labs/dtm/dtmcli/dtmimp"
"github.com/dtm-labs/dtm/dtmcli/logger"
)
// GetGinApp init and return gin
func GetGinApp() *gin.Engine {
gin.SetMode(gin.ReleaseMode)
app := gin.New()
app.Use(gin.Recovery())
app.Use(func(c *gin.Context) {
body := ""
if c.Request.Body != nil {
rb, err := c.GetRawData()
dtmimp.E2P(err)
if len(rb) > 0 {
body = string(rb)
c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(rb))
}
}
logger.Debugf("begin %s %s body: %s", c.Request.Method, c.Request.URL, body)
c.Next()
})
app.Any("/api/ping", func(c *gin.Context) { c.JSON(200, map[string]interface{}{"msg": "pong"}) })
return app
}
// WrapHandler2 wrap a function te bo the handler of gin request
func WrapHandler2(fn func(*gin.Context) interface{}) gin.HandlerFunc {
return func(c *gin.Context) {
began := time.Now()
var err error
r := func() interface{} {
defer dtmimp.P2E(&err)
return fn(c)
}()
status := http.StatusOK
// in dtm test/busi, there are some functions, which will return a resty response
// pass resty response as gin's response
if resp, ok := r.(*resty.Response); ok {
b := resp.Body()
status = resp.StatusCode()
r = nil
err = json.Unmarshal(b, &r)
}
// error maybe returned in r, assign it to err
if ne, ok := r.(error); ok && err == nil {
err = ne
}
// if err != nil || r == nil. then set the status and dtm_result
// dtm_result is for compatible with version lower than v1.10
// when >= v1.10, result test should base on status, not dtm_result.
result := map[string]interface{}{}
if err != nil {
if errors.Is(err, dtmcli.ErrFailure) {
status = http.StatusConflict
result["dtm_result"] = dtmcli.ResultFailure
} else if errors.Is(err, dtmcli.ErrOngoing) {
status = http.StatusTooEarly
result["dtm_result"] = dtmcli.ResultOngoing
} else if err != nil {
status = http.StatusInternalServerError
}
result["message"] = err.Error()
r = result
} else if r == nil {
result["dtm_result"] = dtmcli.ResultSuccess
r = result
}
b, _ := json.Marshal(r)
cont := string(b)
if status == http.StatusOK || status == http.StatusTooEarly {
logger.Infof("%2dms %d %s %s %s", time.Since(began).Milliseconds(), status, c.Request.Method, c.Request.RequestURI, cont)
} else {
logger.Errorf("%2dms %d %s %s %s", time.Since(began).Milliseconds(), status, c.Request.Method, c.Request.RequestURI, cont)
}
c.JSON(status, r)
}
}
// MustGetwd must version of os.Getwd
func MustGetwd() string {
wd, err := os.Getwd()
dtmimp.E2P(err)
return wd
}
// GetSQLDir 获取调用该函数的caller源代码的目录,主要用于测试时,查找相关文件
func GetSQLDir() string {
wd := MustGetwd()
if filepath.Base(wd) == "test" {
wd = filepath.Dir(wd)
}
return wd + "/sqls"
}
// RecoverPanic execs recovery operation
func RecoverPanic(err *error) {
if x := recover(); x != nil {
e := dtmimp.AsError(x)
if err != nil {
*err = e
}
}
}
// GetNextTime gets next time from second
func GetNextTime(second int64) *time.Time {
next := time.Now().Add(time.Duration(second) * time.Second)
return &next
}
// RunSQLScript 1
func RunSQLScript(conf dtmcli.DBConf, script string, skipDrop bool) {
con, err := dtmimp.StandaloneDB(conf)
logger.FatalIfError(err)
defer func() { _ = con.Close() }()
content, err := ioutil.ReadFile(script)
logger.FatalIfError(err)
sqls := strings.Split(string(content), ";")
for _, sql := range sqls {
s := strings.TrimSpace(sql)
if s == "" || (skipDrop && strings.Contains(s, "drop")) {
continue
}
_, err = dtmimp.DBExec(con, s)
logger.FatalIfError(err)
logger.Infof("sql scripts finished: %s", s)
}
}

18
common/utils_test.go → dtmutil/utils_test.go

@ -4,7 +4,7 @@
* license that can be found in the LICENSE file.
*/
package common
package dtmutil
import (
"errors"
@ -16,33 +16,33 @@ import (
"testing"
"github.com/gin-gonic/gin"
"github.com/go-playground/assert/v2"
"github.com/stretchr/testify/assert"
)
func TestGin(t *testing.T) {
app := GetGinApp()
app.GET("/api/sample", WrapHandler(func(c *gin.Context) (interface{}, error) {
return 1, nil
app.GET("/api/sample", WrapHandler2(func(c *gin.Context) interface{} {
return 1
}))
app.GET("/api/error", WrapHandler(func(c *gin.Context) (interface{}, error) {
return nil, errors.New("err1")
app.GET("/api/error", WrapHandler2(func(c *gin.Context) interface{} {
return errors.New("err1")
}))
getResultString := func(api string, body io.Reader) string {
req, _ := http.NewRequest("GET", api, body)
w := httptest.NewRecorder()
app.ServeHTTP(w, req)
return string(w.Body.Bytes())
return w.Body.String()
}
assert.Equal(t, "{\"msg\":\"pong\"}", getResultString("/api/ping", nil))
assert.Equal(t, "1", getResultString("/api/sample", nil))
assert.Equal(t, "{\"code\":500,\"message\":\"err1\"}", getResultString("/api/error", strings.NewReader("{}")))
assert.Equal(t, "{\"message\":\"err1\"}", getResultString("/api/error", strings.NewReader("{}")))
}
func TestFuncs(t *testing.T) {
wd := MustGetwd()
assert.NotEqual(t, "", wd)
dir1 := GetSqlDir()
dir1 := GetSQLDir()
assert.Equal(t, true, strings.HasSuffix(dir1, "/sqls"))
}

14
examples/base.go

@ -1,14 +0,0 @@
package examples
import "fmt"
func Startup() {
InitConfig()
GrpcStartup()
BaseAppStartup()
}
func InitConfig() {
DtmHttpServer = fmt.Sprintf("http://localhost:%d/api/dtmsvr", config.HttpPort)
DtmGrpcServer = fmt.Sprintf("localhost:%d", config.GrpcPort)
}

192
examples/base_http.go

@ -1,192 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package examples
import (
"database/sql"
"errors"
"fmt"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
const (
// BusiAPI busi api prefix
BusiAPI = "/api/busi"
// BusiPort busi server port
BusiPort = 8081
// BusiGrpcPort busi server port
BusiGrpcPort = 58081
)
type setupFunc func(*gin.Engine)
var setupFuncs = map[string]setupFunc{}
// Busi busi service url prefix
var Busi string = fmt.Sprintf("http://localhost:%d%s", BusiPort, BusiAPI)
// BaseAppStartup base app startup
func BaseAppStartup() *gin.Engine {
dtmimp.Logf("examples starting")
app := common.GetGinApp()
app.Use(func(c *gin.Context) {
v := MainSwitch.NextResult.Fetch()
if v != "" {
c.JSON(200, gin.H{"dtm_result": v})
c.Abort()
return
}
c.Next()
})
BaseAddRoute(app)
for k, v := range setupFuncs {
dtmimp.Logf("initing %s", k)
v(app)
}
dtmimp.Logf("Starting busi at: %d", BusiPort)
go app.Run(fmt.Sprintf(":%d", BusiPort))
time.Sleep(100 * time.Millisecond)
return app
}
// AutoEmptyString auto reset to empty when used once
type AutoEmptyString struct {
value string
}
// SetOnce set a value once
func (s *AutoEmptyString) SetOnce(v string) {
s.value = v
}
// Fetch fetch the stored value, then reset the value to empty
func (s *AutoEmptyString) Fetch() string {
v := s.value
s.value = ""
return v
}
type mainSwitchType struct {
TransInResult AutoEmptyString
TransOutResult AutoEmptyString
TransInConfirmResult AutoEmptyString
TransOutConfirmResult AutoEmptyString
TransInRevertResult AutoEmptyString
TransOutRevertResult AutoEmptyString
CanSubmitResult AutoEmptyString
NextResult AutoEmptyString
}
// MainSwitch controls busi success or fail
var MainSwitch mainSwitchType
func handleGeneralBusiness(c *gin.Context, result1 string, result2 string, busi string) (interface{}, error) {
info := infoFromContext(c)
res := dtmimp.OrString(result1, result2, dtmcli.ResultSuccess)
dtmimp.Logf("%s %s result: %s", busi, info.String(), res)
if res == "ERROR" {
return nil, errors.New("ERROR from user")
}
return map[string]interface{}{"dtm_result": res}, nil
}
func error2Resp(err error) (interface{}, error) {
if err != nil {
s := err.Error()
if strings.Contains(s, dtmcli.ResultFailure) || strings.Contains(s, dtmcli.ResultOngoing) {
return gin.H{"dtm_result": s}, nil
}
}
return nil, err
}
// BaseAddRoute add base route handler
func BaseAddRoute(app *gin.Engine) {
app.POST(BusiAPI+"/TransIn", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
return handleGeneralBusiness(c, MainSwitch.TransInResult.Fetch(), reqFrom(c).TransInResult, "transIn")
}))
app.POST(BusiAPI+"/TransOut", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
return handleGeneralBusiness(c, MainSwitch.TransOutResult.Fetch(), reqFrom(c).TransOutResult, "TransOut")
}))
app.POST(BusiAPI+"/TransInConfirm", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
return handleGeneralBusiness(c, MainSwitch.TransInConfirmResult.Fetch(), "", "TransInConfirm")
}))
app.POST(BusiAPI+"/TransOutConfirm", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
return handleGeneralBusiness(c, MainSwitch.TransOutConfirmResult.Fetch(), "", "TransOutConfirm")
}))
app.POST(BusiAPI+"/TransInRevert", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
return handleGeneralBusiness(c, MainSwitch.TransInRevertResult.Fetch(), "", "TransInRevert")
}))
app.POST(BusiAPI+"/TransOutRevert", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
return handleGeneralBusiness(c, MainSwitch.TransOutRevertResult.Fetch(), "", "TransOutRevert")
}))
app.GET(BusiAPI+"/CanSubmit", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
dtmimp.Logf("%s CanSubmit", c.Query("gid"))
return dtmimp.OrString(MainSwitch.CanSubmitResult.Fetch(), dtmcli.ResultSuccess), nil
}))
app.POST(BusiAPI+"/TransInXa", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
err := XaClient.XaLocalTransaction(c.Request.URL.Query(), func(db *sql.DB, xa *dtmcli.Xa) error {
if reqFrom(c).TransInResult == dtmcli.ResultFailure {
return dtmcli.ErrFailure
}
_, err := dtmimp.DBExec(db, "update dtm_busi.user_account set balance=balance+? where user_id=?", reqFrom(c).Amount, 2)
return err
})
return error2Resp(err)
}))
app.POST(BusiAPI+"/TransOutXa", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
err := XaClient.XaLocalTransaction(c.Request.URL.Query(), func(db *sql.DB, xa *dtmcli.Xa) error {
if reqFrom(c).TransOutResult == dtmcli.ResultFailure {
return dtmcli.ErrFailure
}
_, err := dtmimp.DBExec(db, "update dtm_busi.user_account set balance=balance-? where user_id=?", reqFrom(c).Amount, 1)
return err
})
return error2Resp(err)
}))
app.POST(BusiAPI+"/TransOutXaGorm", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
err := XaClient.XaLocalTransaction(c.Request.URL.Query(), func(db *sql.DB, xa *dtmcli.Xa) error {
if reqFrom(c).TransOutResult == dtmcli.ResultFailure {
return dtmcli.ErrFailure
}
var dia gorm.Dialector = nil
if dtmcli.GetCurrentDBType() == dtmcli.DBTypeMysql {
dia = mysql.New(mysql.Config{Conn: db})
} else if dtmcli.GetCurrentDBType() == dtmcli.DBTypePostgres {
dia = postgres.New(postgres.Config{Conn: db})
}
gdb, err := gorm.Open(dia, &gorm.Config{})
if err != nil {
return err
}
dbr := gdb.Exec("update dtm_busi.user_account set balance=balance-? where user_id=?", reqFrom(c).Amount, 1)
return dbr.Error
})
return error2Resp(err)
}))
app.POST(BusiAPI+"/TestPanic", common.WrapHandler(func(c *gin.Context) (interface{}, error) {
if c.Query("panic_error") != "" {
panic(errors.New("panic_error"))
} else if c.Query("panic_string") != "" {
panic("panic_string")
}
return "SUCCESS", nil
}))
}

107
examples/base_types.go

@ -1,107 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package examples
import (
"context"
"database/sql"
"fmt"
"github.com/gin-gonic/gin"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli"
"github.com/yedf/dtm/dtmcli/dtmimp"
"github.com/yedf/dtm/dtmgrpc"
)
// DtmHttpServer dtm service address
var DtmHttpServer = fmt.Sprintf("http://localhost:%d/api/dtmsvr", 36789)
// DtmGrpcServer dtm grpc service address
var DtmGrpcServer = fmt.Sprintf("localhost:%d", 36790)
// TransReq transaction request payload
type TransReq struct {
Amount int `json:"amount"`
TransInResult string `json:"transInResult"`
TransOutResult string `json:"transOutResult"`
}
func (t *TransReq) String() string {
return fmt.Sprintf("amount: %d transIn: %s transOut: %s", t.Amount, t.TransInResult, t.TransOutResult)
}
// GenTransReq 1
func GenTransReq(amount int, outFailed bool, inFailed bool) *TransReq {
return &TransReq{
Amount: amount,
TransOutResult: dtmimp.If(outFailed, dtmcli.ResultFailure, "").(string),
TransInResult: dtmimp.If(inFailed, dtmcli.ResultFailure, "").(string),
}
}
// GenBusiReq 1
func GenBusiReq(amount int, outFailed bool, inFailed bool) *BusiReq {
return &BusiReq{
Amount: int64(amount),
TransOutResult: dtmimp.If(outFailed, dtmcli.ResultFailure, "").(string),
TransInResult: dtmimp.If(inFailed, dtmcli.ResultFailure, "").(string),
}
}
func reqFrom(c *gin.Context) *TransReq {
v, ok := c.Get("trans_req")
if !ok {
req := TransReq{}
err := c.BindJSON(&req)
dtmimp.FatalIfError(err)
c.Set("trans_req", &req)
v = &req
}
return v.(*TransReq)
}
func infoFromContext(c *gin.Context) *dtmcli.BranchBarrier {
info := dtmcli.BranchBarrier{
TransType: c.Query("trans_type"),
Gid: c.Query("gid"),
BranchID: c.Query("branch_id"),
Op: c.Query("op"),
}
return &info
}
func dbGet() *common.DB {
return common.DbGet(config.ExamplesDB)
}
func sdbGet() *sql.DB {
db, err := dtmimp.PooledDB(config.ExamplesDB)
dtmimp.FatalIfError(err)
return db
}
func txGet() *sql.Tx {
db := sdbGet()
tx, err := db.Begin()
dtmimp.FatalIfError(err)
return tx
}
// MustBarrierFromGin 1
func MustBarrierFromGin(c *gin.Context) *dtmcli.BranchBarrier {
ti, err := dtmcli.BarrierFromQuery(c.Request.URL.Query())
dtmimp.FatalIfError(err)
return ti
}
// MustBarrierFromGrpc 1
func MustBarrierFromGrpc(ctx context.Context) *dtmcli.BranchBarrier {
ti, err := dtmgrpc.BarrierFromGrpc(ctx)
dtmimp.FatalIfError(err)
return ti
}

330
examples/busi.pb.go

@ -1,330 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc v3.17.3
// source: examples/busi.proto
package examples
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
emptypb "google.golang.org/protobuf/types/known/emptypb"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// DtmRequest request sent to dtm server
type BusiReq struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Amount int64 `protobuf:"varint,1,opt,name=Amount,proto3" json:"Amount,omitempty"`
TransOutResult string `protobuf:"bytes,2,opt,name=TransOutResult,proto3" json:"TransOutResult,omitempty"`
TransInResult string `protobuf:"bytes,3,opt,name=TransInResult,proto3" json:"TransInResult,omitempty"`
}
func (x *BusiReq) Reset() {
*x = BusiReq{}
if protoimpl.UnsafeEnabled {
mi := &file_examples_busi_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *BusiReq) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*BusiReq) ProtoMessage() {}
func (x *BusiReq) ProtoReflect() protoreflect.Message {
mi := &file_examples_busi_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use BusiReq.ProtoReflect.Descriptor instead.
func (*BusiReq) Descriptor() ([]byte, []int) {
return file_examples_busi_proto_rawDescGZIP(), []int{0}
}
func (x *BusiReq) GetAmount() int64 {
if x != nil {
return x.Amount
}
return 0
}
func (x *BusiReq) GetTransOutResult() string {
if x != nil {
return x.TransOutResult
}
return ""
}
func (x *BusiReq) GetTransInResult() string {
if x != nil {
return x.TransInResult
}
return ""
}
type BusiReply struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
}
func (x *BusiReply) Reset() {
*x = BusiReply{}
if protoimpl.UnsafeEnabled {
mi := &file_examples_busi_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *BusiReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*BusiReply) ProtoMessage() {}
func (x *BusiReply) ProtoReflect() protoreflect.Message {
mi := &file_examples_busi_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use BusiReply.ProtoReflect.Descriptor instead.
func (*BusiReply) Descriptor() ([]byte, []int) {
return file_examples_busi_proto_rawDescGZIP(), []int{1}
}
func (x *BusiReply) GetMessage() string {
if x != nil {
return x.Message
}
return ""
}
var File_examples_busi_proto protoreflect.FileDescriptor
var file_examples_busi_proto_rawDesc = []byte{
0x0a, 0x13, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x62, 0x75, 0x73, 0x69, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x1a,
0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6f, 0x0a, 0x07,
0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x41, 0x6d, 0x6f, 0x75, 0x6e,
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x41, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x12,
0x26, 0x0a, 0x0e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c,
0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x75,
0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x6e, 0x73,
0x49, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d,
0x54, 0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x25, 0x0a,
0x09, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65,
0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73,
0x73, 0x61, 0x67, 0x65, 0x32, 0x97, 0x08, 0x0a, 0x04, 0x42, 0x75, 0x73, 0x69, 0x12, 0x35, 0x0a,
0x09, 0x43, 0x61, 0x6e, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61,
0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x13, 0x2e,
0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x70,
0x6c, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x07, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e, 0x12,
0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52,
0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x08,
0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70,
0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e,
0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65,
0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x0e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x52,
0x65, 0x76, 0x65, 0x72, 0x74, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73,
0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
0x22, 0x00, 0x12, 0x3d, 0x0a, 0x0e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e, 0x43, 0x6f, 0x6e,
0x66, 0x69, 0x72, 0x6d, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e,
0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22,
0x00, 0x12, 0x3e, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x43, 0x6f, 0x6e,
0x66, 0x69, 0x72, 0x6d, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e,
0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22,
0x00, 0x12, 0x3c, 0x0a, 0x08, 0x58, 0x61, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x12, 0x16, 0x2e,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12,
0x38, 0x0a, 0x09, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e, 0x58, 0x61, 0x12, 0x11, 0x2e, 0x65,
0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0a, 0x54, 0x72, 0x61,
0x6e, 0x73, 0x4f, 0x75, 0x74, 0x58, 0x61, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c,
0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
0x74, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e, 0x54,
0x63, 0x63, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75,
0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12,
0x3a, 0x0a, 0x0b, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x54, 0x63, 0x63, 0x12, 0x11,
0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65,
0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x10, 0x54,
0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e, 0x54, 0x63, 0x63, 0x4e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x12,
0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52,
0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x0c,
0x54, 0x72, 0x61, 0x6e, 0x73, 0x49, 0x6e, 0x42, 0x53, 0x61, 0x67, 0x61, 0x12, 0x11, 0x2e, 0x65,
0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0d, 0x54, 0x72, 0x61,
0x6e, 0x73, 0x4f, 0x75, 0x74, 0x42, 0x53, 0x61, 0x67, 0x61, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61,
0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x6e, 0x73,
0x49, 0x6e, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x42, 0x53, 0x61, 0x67, 0x61, 0x12, 0x11, 0x2e,
0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73, 0x69, 0x52, 0x65, 0x71,
0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x13, 0x54, 0x72,
0x61, 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x42, 0x53, 0x61, 0x67,
0x61, 0x12, 0x11, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x73,
0x69, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x42, 0x0c,
0x5a, 0x0a, 0x2e, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (
file_examples_busi_proto_rawDescOnce sync.Once
file_examples_busi_proto_rawDescData = file_examples_busi_proto_rawDesc
)
func file_examples_busi_proto_rawDescGZIP() []byte {
file_examples_busi_proto_rawDescOnce.Do(func() {
file_examples_busi_proto_rawDescData = protoimpl.X.CompressGZIP(file_examples_busi_proto_rawDescData)
})
return file_examples_busi_proto_rawDescData
}
var file_examples_busi_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_examples_busi_proto_goTypes = []interface{}{
(*BusiReq)(nil), // 0: examples.BusiReq
(*BusiReply)(nil), // 1: examples.BusiReply
(*emptypb.Empty)(nil), // 2: google.protobuf.Empty
}
var file_examples_busi_proto_depIdxs = []int32{
0, // 0: examples.Busi.CanSubmit:input_type -> examples.BusiReq
0, // 1: examples.Busi.TransIn:input_type -> examples.BusiReq
0, // 2: examples.Busi.TransOut:input_type -> examples.BusiReq
0, // 3: examples.Busi.TransInRevert:input_type -> examples.BusiReq
0, // 4: examples.Busi.TransOutRevert:input_type -> examples.BusiReq
0, // 5: examples.Busi.TransInConfirm:input_type -> examples.BusiReq
0, // 6: examples.Busi.TransOutConfirm:input_type -> examples.BusiReq
2, // 7: examples.Busi.XaNotify:input_type -> google.protobuf.Empty
0, // 8: examples.Busi.TransInXa:input_type -> examples.BusiReq
0, // 9: examples.Busi.TransOutXa:input_type -> examples.BusiReq
0, // 10: examples.Busi.TransInTcc:input_type -> examples.BusiReq
0, // 11: examples.Busi.TransOutTcc:input_type -> examples.BusiReq
0, // 12: examples.Busi.TransInTccNested:input_type -> examples.BusiReq
0, // 13: examples.Busi.TransInBSaga:input_type -> examples.BusiReq
0, // 14: examples.Busi.TransOutBSaga:input_type -> examples.BusiReq
0, // 15: examples.Busi.TransInRevertBSaga:input_type -> examples.BusiReq
0, // 16: examples.Busi.TransOutRevertBSaga:input_type -> examples.BusiReq
1, // 17: examples.Busi.CanSubmit:output_type -> examples.BusiReply
2, // 18: examples.Busi.TransIn:output_type -> google.protobuf.Empty
2, // 19: examples.Busi.TransOut:output_type -> google.protobuf.Empty
2, // 20: examples.Busi.TransInRevert:output_type -> google.protobuf.Empty
2, // 21: examples.Busi.TransOutRevert:output_type -> google.protobuf.Empty
2, // 22: examples.Busi.TransInConfirm:output_type -> google.protobuf.Empty
2, // 23: examples.Busi.TransOutConfirm:output_type -> google.protobuf.Empty
2, // 24: examples.Busi.XaNotify:output_type -> google.protobuf.Empty
2, // 25: examples.Busi.TransInXa:output_type -> google.protobuf.Empty
2, // 26: examples.Busi.TransOutXa:output_type -> google.protobuf.Empty
2, // 27: examples.Busi.TransInTcc:output_type -> google.protobuf.Empty
2, // 28: examples.Busi.TransOutTcc:output_type -> google.protobuf.Empty
2, // 29: examples.Busi.TransInTccNested:output_type -> google.protobuf.Empty
2, // 30: examples.Busi.TransInBSaga:output_type -> google.protobuf.Empty
2, // 31: examples.Busi.TransOutBSaga:output_type -> google.protobuf.Empty
2, // 32: examples.Busi.TransInRevertBSaga:output_type -> google.protobuf.Empty
2, // 33: examples.Busi.TransOutRevertBSaga:output_type -> google.protobuf.Empty
17, // [17:34] is the sub-list for method output_type
0, // [0:17] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_examples_busi_proto_init() }
func file_examples_busi_proto_init() {
if File_examples_busi_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_examples_busi_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*BusiReq); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_examples_busi_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*BusiReply); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_examples_busi_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_examples_busi_proto_goTypes,
DependencyIndexes: file_examples_busi_proto_depIdxs,
MessageInfos: file_examples_busi_proto_msgTypes,
}.Build()
File_examples_busi_proto = out.File
file_examples_busi_proto_rawDesc = nil
file_examples_busi_proto_goTypes = nil
file_examples_busi_proto_depIdxs = nil
}

55
examples/data.go

@ -1,55 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package examples
import (
"fmt"
"github.com/yedf/dtm/common"
"github.com/yedf/dtm/dtmcli/dtmimp"
)
var config = &common.Config
func resetXaData() {
if config.ExamplesDB.Driver != "mysql" {
return
}
db := dbGet()
type XaRow struct {
Data string
}
xas := []XaRow{}
db.Must().Raw("xa recover").Scan(&xas)
for _, xa := range xas {
db.Must().Exec(fmt.Sprintf("xa rollback '%s'", xa.Data))
}
}
// PopulateDB populate example mysql data
func PopulateDB(skipDrop bool) {
resetXaData()
file := fmt.Sprintf("%s/examples.%s.sql", common.GetSqlDir(), config.ExamplesDB.Driver)
common.RunSQLScript(config.ExamplesDB, file, skipDrop)
file = fmt.Sprintf("%s/dtmcli.barrier.%s.sql", common.GetSqlDir(), config.ExamplesDB.Driver)
common.RunSQLScript(config.ExamplesDB, file, skipDrop)
}
type sampleInfo struct {
Arg string
Action func() string
Desc string
}
// Samples 所有的示例都会注册到这里
var Samples = map[string]*sampleInfo{}
func addSample(name string, fn func() string) {
dtmimp.LogIfFatalf(Samples[name] != nil, "%s already exists", name)
Samples[name] = &sampleInfo{Arg: name, Action: fn}
}

25
examples/grpc_msg.go

@ -1,25 +0,0 @@
/*
* Copyright (c) 2021 yedf. All rights reserved.
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file.
*/
package examples
import (
"github.com/yedf/dtm/dtmcli/dtmimp"
dtmgrpc "github.com/yedf/dtm/dtmgrpc"
)
func init() {
addSample("grpc_msg", func() string {
req := &BusiReq{Amount: 30}
gid := dtmgrpc.MustGenGid(DtmGrpcServer)
msg := dtmgrpc.NewMsgGrpc(DtmGrpcServer, gid).
Add(BusiGrpc+"/examples.Busi/TransOut", req).
Add(BusiGrpc+"/examples.Busi/TransIn", req)
err := msg.Submit()
dtmimp.FatalIfError(err)
return msg.Gid
})
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save