tsai
/
dtm
mirror of https://github.com/dtm-labs/dtm.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #471 from dtm-labs/alpha 

use a new algorithm to do lock one trans
pull/473/head v1.17.8
yedf2 2 years ago
committed by GitHub
parent
e7047333cd a5f0ae1391
commit
dedf72ea95
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 67 additions and 60 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 92
      dtmsvr/storage/boltdb/boltdb_test.go
  2. 23
      dtmsvr/storage/sql/sql.go
  3. 12
      dtmsvr/trans_status.go

92
dtmsvr/storage/boltdb/boltdb_test.go
View File

@ -11,7 +11,7 @@ import (
"testing" "testing"
"time" "time"
. "github.com/onsi/gomega" ga "github.com/onsi/gomega"
bolt "go.etcd.io/bbolt" bolt "go.etcd.io/bbolt"
"github.com/dtm-labs/dtm/client/dtmcli/dtmimp" "github.com/dtm-labs/dtm/client/dtmcli/dtmimp"
@ -20,13 +20,13 @@ import (
func TestInitializeBuckets(t *testing.T) { func TestInitializeBuckets(t *testing.T) {
t.Run("normal test", func(t *testing.T) { t.Run("normal test", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
err = initializeBuckets(db) err = initializeBuckets(db)
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
actualBuckets := [][]byte{} actualBuckets := [][]byte{}
err = db.View(func(t *bolt.Tx) error { err = db.View(func(t *bolt.Tx) error {
@ -35,42 +35,42 @@ func TestInitializeBuckets(t *testing.T) {
return nil return nil
}) })
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
g.Expect(actualBuckets).To(Equal(allBuckets)) g.Expect(actualBuckets).To(ga.Equal(allBuckets))
}) })
} }
func TestCleanupExpiredData(t *testing.T) { func TestCleanupExpiredData(t *testing.T) {
t.Run("negative expired seconds", func(t *testing.T) { t.Run("negative expired seconds", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
err = cleanupExpiredData(-1*time.Second, db) err = cleanupExpiredData(-1*time.Second, db)
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
}) })
t.Run("nil global bucket", func(t *testing.T) { t.Run("nil global bucket", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
err = cleanupExpiredData(time.Second, db) err = cleanupExpiredData(time.Second, db)
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
}) })
t.Run("normal test", func(t *testing.T) { t.Run("normal test", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
// Initialize data // Initialize data
err = initializeBuckets(db) err = initializeBuckets(db)
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
err = db.Update(func(t *bolt.Tx) error { err = db.Update(func(t *bolt.Tx) error {
doneTime := time.Now().Add(-10 * time.Minute) doneTime := time.Now().Add(-10 * time.Minute)
@ -95,10 +95,10 @@ func TestCleanupExpiredData(t *testing.T) {
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
err = cleanupExpiredData(time.Minute, db) err = cleanupExpiredData(time.Minute, db)
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
actualGids := []string{} actualGids := []string{}
err = db.View(func(t *bolt.Tx) error { err = db.View(func(t *bolt.Tx) error {
@ -108,29 +108,29 @@ func TestCleanupExpiredData(t *testing.T) {
} }
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
g.Expect(actualGids).To(Equal([]string{"gid0"})) g.Expect(actualGids).To(ga.Equal([]string{"gid0"}))
}) })
} }
func TestCleanupGlobalWithGids(t *testing.T) { func TestCleanupGlobalWithGids(t *testing.T) {
t.Run("nil bucket", func(t *testing.T) { t.Run("nil bucket", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
err = db.Update(func(t *bolt.Tx) error { err = db.Update(func(t *bolt.Tx) error {
cleanupGlobalWithGids(t, nil) cleanupGlobalWithGids(t, nil)
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
}) })
t.Run("normal test", func(t *testing.T) { t.Run("normal test", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
// Initialize data // Initialize data
@ -151,7 +151,7 @@ func TestCleanupGlobalWithGids(t *testing.T) {
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
err = db.Update(func(t *bolt.Tx) error { err = db.Update(func(t *bolt.Tx) error {
cleanupGlobalWithGids(t, map[string]struct{}{ cleanupGlobalWithGids(t, map[string]struct{}{
@ -160,7 +160,7 @@ func TestCleanupGlobalWithGids(t *testing.T) {
}) })
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
actualGids := []string{} actualGids := []string{}
err = db.View(func(t *bolt.Tx) error { err = db.View(func(t *bolt.Tx) error {
@ -170,29 +170,29 @@ func TestCleanupGlobalWithGids(t *testing.T) {
} }
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
g.Expect(actualGids).To(Equal([]string{"k3"})) g.Expect(actualGids).To(ga.Equal([]string{"k3"}))
}) })
} }
func TestCleanupBranchWithGids(t *testing.T) { func TestCleanupBranchWithGids(t *testing.T) {
t.Run("nil bucket", func(t *testing.T) { t.Run("nil bucket", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
err = db.Update(func(t *bolt.Tx) error { err = db.Update(func(t *bolt.Tx) error {
cleanupBranchWithGids(t, nil) cleanupBranchWithGids(t, nil)
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
}) })
t.Run("normal test", func(t *testing.T) { t.Run("normal test", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
// Initialize data // Initialize data
@ -232,7 +232,7 @@ func TestCleanupBranchWithGids(t *testing.T) {
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
err = db.Update(func(t *bolt.Tx) error { err = db.Update(func(t *bolt.Tx) error {
cleanupBranchWithGids(t, map[string]struct{}{ cleanupBranchWithGids(t, map[string]struct{}{
@ -241,7 +241,7 @@ func TestCleanupBranchWithGids(t *testing.T) {
}) })
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
actualKeys := []string{} actualKeys := []string{}
err = db.View(func(t *bolt.Tx) error { err = db.View(func(t *bolt.Tx) error {
@ -251,29 +251,29 @@ func TestCleanupBranchWithGids(t *testing.T) {
} }
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
g.Expect(actualKeys).To(Equal([]string{"a", "gid201", "z"})) g.Expect(actualKeys).To(ga.Equal([]string{"a", "gid201", "z"}))
}) })
} }
func TestCleanupIndexWithGids(t *testing.T) { func TestCleanupIndexWithGids(t *testing.T) {
t.Run("nil bucket", func(t *testing.T) { t.Run("nil bucket", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
err = db.Update(func(t *bolt.Tx) error { err = db.Update(func(t *bolt.Tx) error {
cleanupIndexWithGids(t, nil) cleanupIndexWithGids(t, nil)
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
}) })
t.Run("normal test", func(t *testing.T) { t.Run("normal test", func(t *testing.T) {
g := NewWithT(t) g := ga.NewWithT(t)
db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second}) db, err := bolt.Open(path.Join(t.TempDir(), "./test.bolt"), 0666, &bolt.Options{Timeout: 1 * time.Second})
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
defer db.Close() defer db.Close()
// Initialize data // Initialize data
@ -313,7 +313,7 @@ func TestCleanupIndexWithGids(t *testing.T) {
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
err = db.Update(func(t *bolt.Tx) error { err = db.Update(func(t *bolt.Tx) error {
cleanupIndexWithGids(t, map[string]struct{}{ cleanupIndexWithGids(t, map[string]struct{}{
@ -322,7 +322,7 @@ func TestCleanupIndexWithGids(t *testing.T) {
}) })
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
actualKeys := []string{} actualKeys := []string{}
err = db.View(func(t *bolt.Tx) error { err = db.View(func(t *bolt.Tx) error {
@ -332,7 +332,7 @@ func TestCleanupIndexWithGids(t *testing.T) {
} }
return nil return nil
}) })
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(ga.HaveOccurred())
g.Expect(actualKeys).To(Equal([]string{"3-gid2", "a", "z"})) g.Expect(actualKeys).To(ga.Equal([]string{"3-gid2", "a", "z"}))
}) })
} }

23
dtmsvr/storage/sql/sql.go
View File

@ -8,6 +8,8 @@
package sql package sql
import ( import (
"database/sql"
"errors"
"fmt" "fmt"
"math" "math"
"time" "time"
@ -159,23 +161,26 @@ func (s *Store) LockOneGlobalTrans(expireIn time.Duration) *storage.TransGlobalS
db := dbGet() db := dbGet()
owner := shortuuid.New() owner := shortuuid.New()
nextCronTime := getTimeStr(int64(expireIn / time.Second)) nextCronTime := getTimeStr(int64(expireIn / time.Second))
where := map[string]string{ where := fmt.Sprintf(`next_cron_time < '%s' and status in ('prepared', 'aborting', 'submitted')`, nextCronTime)
dtmimp.DBTypeMysql: fmt.Sprintf(`next_cron_time < '%s' and status in ('prepared', 'aborting', 'submitted') limit 1`, nextCronTime),
dtmimp.DBTypePostgres: fmt.Sprintf(`id in (select id from trans_global where next_cron_time < '%s' and status in ('prepared', 'aborting', 'submitted') limit 1 )`, nextCronTime), order := map[string]string{
dtmimp.DBTypeMysql: `order by rand()`,
dtmimp.DBTypePostgres: `order by random()`,
}[conf.Store.Driver] }[conf.Store.Driver]
ssql := fmt.Sprintf(`select count(1) from trans_global where %s`, where) ssql := fmt.Sprintf(`select id from trans_global where %s %s limit 1`, where, order)
var cnt int64 var id int64
err := db.ToSQLDB().QueryRow(ssql).Scan(&cnt) err := db.ToSQLDB().QueryRow(ssql).Scan(&id)
dtmimp.PanicIf(err != nil, err) if errors.Is(err, sql.ErrNoRows) {
if cnt == 0 {
return nil return nil
} }
dtmimp.PanicIf(err != nil, err)
sql := fmt.Sprintf(`UPDATE trans_global SET update_time='%s',next_cron_time='%s', owner='%s' WHERE %s`, sql := fmt.Sprintf(`UPDATE trans_global SET update_time='%s',next_cron_time='%s', owner='%s' WHERE id=%d and %s`,
getTimeStr(0), getTimeStr(0),
getTimeStr(conf.RetryInterval), getTimeStr(conf.RetryInterval),
owner, owner,
id,
where) where)
affected, err := dtmimp.DBExec(conf.Store.Driver, db.ToSQLDB(), sql) affected, err := dtmimp.DBExec(conf.Store.Driver, db.ToSQLDB(), sql)

12
dtmsvr/trans_status.go
View File

@ -262,13 +262,15 @@ func (t *TransGlobal) execBranch(ctx context.Context, branch *TransBranch, branc
func (t *TransGlobal) getNextCronInterval(ctype cronType) int64 { func (t *TransGlobal) getNextCronInterval(ctype cronType) int64 {
if ctype == cronBackoff { if ctype == cronBackoff {
return t.NextCronInterval * 2 return t.NextCronInterval * 2
} else if ctype == cronKeep { }
if ctype == cronKeep {
return t.NextCronInterval return t.NextCronInterval
} else if t.RetryInterval != 0 { }
if t.RetryInterval != 0 {
return t.RetryInterval return t.RetryInterval
} else if t.TimeoutToFail > 0 && t.TimeoutToFail < conf.RetryInterval { }
if t.TimeoutToFail > 0 && t.TimeoutToFail < conf.RetryInterval {
return t.TimeoutToFail return t.TimeoutToFail
} else {
return conf.RetryInterval
} }
return conf.RetryInterval
} }

Write Preview
Loading…
Cancel
Save