initial import
This commit is contained in:
229
core/breaker/breaker.go
Normal file
229
core/breaker/breaker.go
Normal file
@@ -0,0 +1,229 @@
|
||||
package breaker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"zero/core/mathx"
|
||||
"zero/core/proc"
|
||||
"zero/core/stat"
|
||||
"zero/core/stringx"
|
||||
)
|
||||
|
||||
const (
|
||||
StateClosed State = iota
|
||||
StateOpen
|
||||
)
|
||||
|
||||
const (
|
||||
numHistoryReasons = 5
|
||||
timeFormat = "15:04:05"
|
||||
)
|
||||
|
||||
// ErrServiceUnavailable is returned when the CB state is open
|
||||
var ErrServiceUnavailable = errors.New("circuit breaker is open")
|
||||
|
||||
type (
|
||||
State = int32
|
||||
Acceptable func(err error) bool
|
||||
|
||||
Breaker interface {
|
||||
// Name returns the name of the netflixBreaker.
|
||||
Name() string
|
||||
|
||||
// Allow checks if the request is allowed.
|
||||
// If allowed, a promise will be returned, the caller needs to call promise.Accept()
|
||||
// on success, or call promise.Reject() on failure.
|
||||
// If not allow, ErrServiceUnavailable will be returned.
|
||||
Allow() (Promise, error)
|
||||
|
||||
// Do runs the given request if the netflixBreaker accepts it.
|
||||
// Do returns an error instantly if the netflixBreaker rejects the request.
|
||||
// If a panic occurs in the request, the netflixBreaker handles it as an error
|
||||
// and causes the same panic again.
|
||||
Do(req func() error) error
|
||||
|
||||
// DoWithAcceptable runs the given request if the netflixBreaker accepts it.
|
||||
// Do returns an error instantly if the netflixBreaker rejects the request.
|
||||
// If a panic occurs in the request, the netflixBreaker handles it as an error
|
||||
// and causes the same panic again.
|
||||
// acceptable checks if it's a successful call, even if the err is not nil.
|
||||
DoWithAcceptable(req func() error, acceptable Acceptable) error
|
||||
|
||||
// DoWithFallback runs the given request if the netflixBreaker accepts it.
|
||||
// DoWithFallback runs the fallback if the netflixBreaker rejects the request.
|
||||
// If a panic occurs in the request, the netflixBreaker handles it as an error
|
||||
// and causes the same panic again.
|
||||
DoWithFallback(req func() error, fallback func(err error) error) error
|
||||
|
||||
// DoWithFallbackAcceptable runs the given request if the netflixBreaker accepts it.
|
||||
// DoWithFallback runs the fallback if the netflixBreaker rejects the request.
|
||||
// If a panic occurs in the request, the netflixBreaker handles it as an error
|
||||
// and causes the same panic again.
|
||||
// acceptable checks if it's a successful call, even if the err is not nil.
|
||||
DoWithFallbackAcceptable(req func() error, fallback func(err error) error, acceptable Acceptable) error
|
||||
}
|
||||
|
||||
BreakerOption func(breaker *circuitBreaker)
|
||||
|
||||
Promise interface {
|
||||
Accept()
|
||||
Reject(reason string)
|
||||
}
|
||||
|
||||
internalPromise interface {
|
||||
Accept()
|
||||
Reject()
|
||||
}
|
||||
|
||||
circuitBreaker struct {
|
||||
name string
|
||||
throttle
|
||||
}
|
||||
|
||||
internalThrottle interface {
|
||||
allow() (internalPromise, error)
|
||||
doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error
|
||||
}
|
||||
|
||||
throttle interface {
|
||||
allow() (Promise, error)
|
||||
doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error
|
||||
}
|
||||
)
|
||||
|
||||
func NewBreaker(opts ...BreakerOption) Breaker {
|
||||
var b circuitBreaker
|
||||
for _, opt := range opts {
|
||||
opt(&b)
|
||||
}
|
||||
if len(b.name) == 0 {
|
||||
b.name = stringx.Rand()
|
||||
}
|
||||
b.throttle = newLoggedThrottle(b.name, newGoogleBreaker())
|
||||
|
||||
return &b
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) Allow() (Promise, error) {
|
||||
return cb.throttle.allow()
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) Do(req func() error) error {
|
||||
return cb.throttle.doReq(req, nil, defaultAcceptable)
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) DoWithAcceptable(req func() error, acceptable Acceptable) error {
|
||||
return cb.throttle.doReq(req, nil, acceptable)
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
|
||||
return cb.throttle.doReq(req, fallback, defaultAcceptable)
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
|
||||
acceptable Acceptable) error {
|
||||
return cb.throttle.doReq(req, fallback, acceptable)
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) Name() string {
|
||||
return cb.name
|
||||
}
|
||||
|
||||
func WithName(name string) BreakerOption {
|
||||
return func(b *circuitBreaker) {
|
||||
b.name = name
|
||||
}
|
||||
}
|
||||
|
||||
func defaultAcceptable(err error) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
type loggedThrottle struct {
|
||||
name string
|
||||
internalThrottle
|
||||
errWin *errorWindow
|
||||
}
|
||||
|
||||
func newLoggedThrottle(name string, t internalThrottle) loggedThrottle {
|
||||
return loggedThrottle{
|
||||
name: name,
|
||||
internalThrottle: t,
|
||||
errWin: new(errorWindow),
|
||||
}
|
||||
}
|
||||
|
||||
func (lt loggedThrottle) allow() (Promise, error) {
|
||||
promise, err := lt.internalThrottle.allow()
|
||||
return promiseWithReason{
|
||||
promise: promise,
|
||||
errWin: lt.errWin,
|
||||
}, lt.logError(err)
|
||||
}
|
||||
|
||||
func (lt loggedThrottle) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
|
||||
return lt.logError(lt.internalThrottle.doReq(req, fallback, func(err error) bool {
|
||||
accept := acceptable(err)
|
||||
if !accept {
|
||||
lt.errWin.add(err.Error())
|
||||
}
|
||||
return accept
|
||||
}))
|
||||
}
|
||||
|
||||
func (lt loggedThrottle) logError(err error) error {
|
||||
if err == ErrServiceUnavailable {
|
||||
// if circuit open, not possible to have empty error window
|
||||
stat.Report(fmt.Sprintf(
|
||||
"proc(%s/%d), callee: %s, breaker is open and requests dropped\nlast errors:\n%s",
|
||||
proc.ProcessName(), proc.Pid(), lt.name, lt.errWin))
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
type errorWindow struct {
|
||||
reasons [numHistoryReasons]string
|
||||
index int
|
||||
count int
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
func (ew *errorWindow) add(reason string) {
|
||||
ew.lock.Lock()
|
||||
ew.reasons[ew.index] = fmt.Sprintf("%s %s", time.Now().Format(timeFormat), reason)
|
||||
ew.index = (ew.index + 1) % numHistoryReasons
|
||||
ew.count = mathx.MinInt(ew.count+1, numHistoryReasons)
|
||||
ew.lock.Unlock()
|
||||
}
|
||||
|
||||
func (ew *errorWindow) String() string {
|
||||
var builder strings.Builder
|
||||
|
||||
ew.lock.Lock()
|
||||
for i := ew.index + ew.count - 1; i >= ew.index; i-- {
|
||||
builder.WriteString(ew.reasons[i%numHistoryReasons])
|
||||
builder.WriteByte('\n')
|
||||
}
|
||||
ew.lock.Unlock()
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
type promiseWithReason struct {
|
||||
promise internalPromise
|
||||
errWin *errorWindow
|
||||
}
|
||||
|
||||
func (p promiseWithReason) Accept() {
|
||||
p.promise.Accept()
|
||||
}
|
||||
|
||||
func (p promiseWithReason) Reject(reason string) {
|
||||
p.errWin.add(reason)
|
||||
p.promise.Reject()
|
||||
}
|
||||
44
core/breaker/breaker_test.go
Normal file
44
core/breaker/breaker_test.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package breaker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"zero/core/stat"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func init() {
|
||||
stat.SetReporter(nil)
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_Allow(t *testing.T) {
|
||||
b := NewBreaker()
|
||||
assert.True(t, len(b.Name()) > 0)
|
||||
_, err := b.Allow()
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestLogReason(t *testing.T) {
|
||||
b := NewBreaker()
|
||||
assert.True(t, len(b.Name()) > 0)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
_ = b.Do(func() error {
|
||||
return errors.New(strconv.Itoa(i))
|
||||
})
|
||||
}
|
||||
errs := b.(*circuitBreaker).throttle.(loggedThrottle).errWin
|
||||
assert.Equal(t, numHistoryReasons, errs.count)
|
||||
}
|
||||
|
||||
func BenchmarkGoogleBreaker(b *testing.B) {
|
||||
br := NewBreaker()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = br.Do(func() error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
76
core/breaker/breakers.go
Normal file
76
core/breaker/breakers.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package breaker
|
||||
|
||||
import "sync"
|
||||
|
||||
var (
|
||||
lock sync.RWMutex
|
||||
breakers = make(map[string]Breaker)
|
||||
)
|
||||
|
||||
func Do(name string, req func() error) error {
|
||||
return do(name, func(b Breaker) error {
|
||||
return b.Do(req)
|
||||
})
|
||||
}
|
||||
|
||||
func DoWithAcceptable(name string, req func() error, acceptable Acceptable) error {
|
||||
return do(name, func(b Breaker) error {
|
||||
return b.DoWithAcceptable(req, acceptable)
|
||||
})
|
||||
}
|
||||
|
||||
func DoWithFallback(name string, req func() error, fallback func(err error) error) error {
|
||||
return do(name, func(b Breaker) error {
|
||||
return b.DoWithFallback(req, fallback)
|
||||
})
|
||||
}
|
||||
|
||||
func DoWithFallbackAcceptable(name string, req func() error, fallback func(err error) error,
|
||||
acceptable Acceptable) error {
|
||||
return do(name, func(b Breaker) error {
|
||||
return b.DoWithFallbackAcceptable(req, fallback, acceptable)
|
||||
})
|
||||
}
|
||||
|
||||
func GetBreaker(name string) Breaker {
|
||||
lock.RLock()
|
||||
b, ok := breakers[name]
|
||||
lock.RUnlock()
|
||||
if ok {
|
||||
return b
|
||||
}
|
||||
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
b = NewBreaker()
|
||||
breakers[name] = b
|
||||
return b
|
||||
}
|
||||
|
||||
func NoBreakFor(name string) {
|
||||
lock.Lock()
|
||||
breakers[name] = newNoOpBreaker()
|
||||
lock.Unlock()
|
||||
}
|
||||
|
||||
func do(name string, execute func(b Breaker) error) error {
|
||||
lock.RLock()
|
||||
b, ok := breakers[name]
|
||||
lock.RUnlock()
|
||||
if ok {
|
||||
return execute(b)
|
||||
} else {
|
||||
lock.Lock()
|
||||
b, ok = breakers[name]
|
||||
if ok {
|
||||
lock.Unlock()
|
||||
return execute(b)
|
||||
} else {
|
||||
b = NewBreaker(WithName(name))
|
||||
breakers[name] = b
|
||||
lock.Unlock()
|
||||
return execute(b)
|
||||
}
|
||||
}
|
||||
}
|
||||
115
core/breaker/breakers_test.go
Normal file
115
core/breaker/breakers_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package breaker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"zero/core/stat"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func init() {
|
||||
stat.SetReporter(nil)
|
||||
}
|
||||
|
||||
func TestBreakersDo(t *testing.T) {
|
||||
assert.Nil(t, Do("any", func() error {
|
||||
return nil
|
||||
}))
|
||||
|
||||
errDummy := errors.New("any")
|
||||
assert.Equal(t, errDummy, Do("any", func() error {
|
||||
return errDummy
|
||||
}))
|
||||
}
|
||||
|
||||
func TestBreakersDoWithAcceptable(t *testing.T) {
|
||||
errDummy := errors.New("anyone")
|
||||
for i := 0; i < 10000; i++ {
|
||||
assert.Equal(t, errDummy, GetBreaker("anyone").DoWithAcceptable(func() error {
|
||||
return errDummy
|
||||
}, func(err error) bool {
|
||||
return err == nil || err == errDummy
|
||||
}))
|
||||
}
|
||||
verify(t, func() bool {
|
||||
return Do("anyone", func() error {
|
||||
return nil
|
||||
}) == nil
|
||||
})
|
||||
|
||||
for i := 0; i < 10000; i++ {
|
||||
err := DoWithAcceptable("another", func() error {
|
||||
return errDummy
|
||||
}, func(err error) bool {
|
||||
return err == nil
|
||||
})
|
||||
assert.True(t, err == errDummy || err == ErrServiceUnavailable)
|
||||
}
|
||||
verify(t, func() bool {
|
||||
return ErrServiceUnavailable == Do("another", func() error {
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestBreakersNoBreakerFor(t *testing.T) {
|
||||
NoBreakFor("any")
|
||||
errDummy := errors.New("any")
|
||||
for i := 0; i < 10000; i++ {
|
||||
assert.Equal(t, errDummy, GetBreaker("any").Do(func() error {
|
||||
return errDummy
|
||||
}))
|
||||
}
|
||||
assert.Equal(t, nil, Do("any", func() error {
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
|
||||
func TestBreakersFallback(t *testing.T) {
|
||||
errDummy := errors.New("any")
|
||||
for i := 0; i < 10000; i++ {
|
||||
err := DoWithFallback("fallback", func() error {
|
||||
return errDummy
|
||||
}, func(err error) error {
|
||||
return nil
|
||||
})
|
||||
assert.True(t, err == nil || err == errDummy)
|
||||
}
|
||||
verify(t, func() bool {
|
||||
return ErrServiceUnavailable == Do("fallback", func() error {
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestBreakersAcceptableFallback(t *testing.T) {
|
||||
errDummy := errors.New("any")
|
||||
for i := 0; i < 10000; i++ {
|
||||
err := DoWithFallbackAcceptable("acceptablefallback", func() error {
|
||||
return errDummy
|
||||
}, func(err error) error {
|
||||
return nil
|
||||
}, func(err error) bool {
|
||||
return err == nil
|
||||
})
|
||||
assert.True(t, err == nil || err == errDummy)
|
||||
}
|
||||
verify(t, func() bool {
|
||||
return ErrServiceUnavailable == Do("acceptablefallback", func() error {
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func verify(t *testing.T, fn func() bool) {
|
||||
var count int
|
||||
for i := 0; i < 100; i++ {
|
||||
if fn() {
|
||||
count++
|
||||
}
|
||||
}
|
||||
assert.True(t, count >= 80, fmt.Sprintf("should be greater than 80, actual %d", count))
|
||||
}
|
||||
125
core/breaker/googlebreaker.go
Normal file
125
core/breaker/googlebreaker.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package breaker
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"zero/core/collection"
|
||||
"zero/core/mathx"
|
||||
)
|
||||
|
||||
const (
|
||||
// 250ms for bucket duration
|
||||
window = time.Second * 10
|
||||
buckets = 40
|
||||
k = 1.5
|
||||
protection = 5
|
||||
)
|
||||
|
||||
// googleBreaker is a netflixBreaker pattern from google.
|
||||
// see Client-Side Throttling section in https://landing.google.com/sre/sre-book/chapters/handling-overload/
|
||||
type googleBreaker struct {
|
||||
k float64
|
||||
state int32
|
||||
stat *collection.RollingWindow
|
||||
proba *mathx.Proba
|
||||
}
|
||||
|
||||
func newGoogleBreaker() *googleBreaker {
|
||||
bucketDuration := time.Duration(int64(window) / int64(buckets))
|
||||
st := collection.NewRollingWindow(buckets, bucketDuration)
|
||||
return &googleBreaker{
|
||||
stat: st,
|
||||
k: k,
|
||||
state: StateClosed,
|
||||
proba: mathx.NewProba(),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *googleBreaker) accept() error {
|
||||
accepts, total := b.history()
|
||||
weightedAccepts := b.k * float64(accepts)
|
||||
// https://landing.google.com/sre/sre-book/chapters/handling-overload/#eq2101
|
||||
dropRatio := math.Max(0, (float64(total-protection)-weightedAccepts)/float64(total+1))
|
||||
if dropRatio <= 0 {
|
||||
if atomic.LoadInt32(&b.state) == StateOpen {
|
||||
atomic.CompareAndSwapInt32(&b.state, StateOpen, StateClosed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if atomic.LoadInt32(&b.state) == StateClosed {
|
||||
atomic.CompareAndSwapInt32(&b.state, StateClosed, StateOpen)
|
||||
}
|
||||
if b.proba.TrueOnProba(dropRatio) {
|
||||
return ErrServiceUnavailable
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *googleBreaker) allow() (internalPromise, error) {
|
||||
if err := b.accept(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return googlePromise{
|
||||
b: b,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *googleBreaker) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
|
||||
if err := b.accept(); err != nil {
|
||||
if fallback != nil {
|
||||
return fallback(err)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
b.markFailure()
|
||||
panic(e)
|
||||
}
|
||||
}()
|
||||
|
||||
err := req()
|
||||
if acceptable(err) {
|
||||
b.markSuccess()
|
||||
} else {
|
||||
b.markFailure()
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *googleBreaker) markSuccess() {
|
||||
b.stat.Add(1)
|
||||
}
|
||||
|
||||
func (b *googleBreaker) markFailure() {
|
||||
b.stat.Add(0)
|
||||
}
|
||||
|
||||
func (b *googleBreaker) history() (accepts int64, total int64) {
|
||||
b.stat.Reduce(func(b *collection.Bucket) {
|
||||
accepts += int64(b.Sum)
|
||||
total += b.Count
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type googlePromise struct {
|
||||
b *googleBreaker
|
||||
}
|
||||
|
||||
func (p googlePromise) Accept() {
|
||||
p.b.markSuccess()
|
||||
}
|
||||
|
||||
func (p googlePromise) Reject() {
|
||||
p.b.markFailure()
|
||||
}
|
||||
238
core/breaker/googlebreaker_test.go
Normal file
238
core/breaker/googlebreaker_test.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package breaker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"zero/core/collection"
|
||||
"zero/core/mathx"
|
||||
"zero/core/stat"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
testBuckets = 10
|
||||
testInterval = time.Millisecond * 10
|
||||
)
|
||||
|
||||
func init() {
|
||||
stat.SetReporter(nil)
|
||||
}
|
||||
|
||||
func getGoogleBreaker() *googleBreaker {
|
||||
st := collection.NewRollingWindow(testBuckets, testInterval)
|
||||
return &googleBreaker{
|
||||
stat: st,
|
||||
k: 5,
|
||||
state: StateClosed,
|
||||
proba: mathx.NewProba(),
|
||||
}
|
||||
}
|
||||
|
||||
func markSuccessWithDuration(b *googleBreaker, count int, sleep time.Duration) {
|
||||
for i := 0; i < count; i++ {
|
||||
b.markSuccess()
|
||||
time.Sleep(sleep)
|
||||
}
|
||||
}
|
||||
|
||||
func markFailedWithDuration(b *googleBreaker, count int, sleep time.Duration) {
|
||||
for i := 0; i < count; i++ {
|
||||
b.markFailure()
|
||||
time.Sleep(sleep)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleBreakerClose(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
markSuccess(b, 80)
|
||||
assert.Nil(t, b.accept())
|
||||
markSuccess(b, 120)
|
||||
assert.Nil(t, b.accept())
|
||||
}
|
||||
|
||||
func TestGoogleBreakerOpen(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
markSuccess(b, 10)
|
||||
assert.Nil(t, b.accept())
|
||||
markFailed(b, 100000)
|
||||
time.Sleep(testInterval * 2)
|
||||
verify(t, func() bool {
|
||||
return b.accept() != nil
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoogleBreakerFallback(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
markSuccess(b, 1)
|
||||
assert.Nil(t, b.accept())
|
||||
markFailed(b, 10000)
|
||||
time.Sleep(testInterval * 2)
|
||||
verify(t, func() bool {
|
||||
return b.doReq(func() error {
|
||||
return errors.New("any")
|
||||
}, func(err error) error {
|
||||
return nil
|
||||
}, defaultAcceptable) == nil
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoogleBreakerReject(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
markSuccess(b, 100)
|
||||
assert.Nil(t, b.accept())
|
||||
markFailed(b, 10000)
|
||||
time.Sleep(testInterval)
|
||||
assert.Equal(t, ErrServiceUnavailable, b.doReq(func() error {
|
||||
return ErrServiceUnavailable
|
||||
}, nil, defaultAcceptable))
|
||||
}
|
||||
|
||||
func TestGoogleBreakerAcceptable(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
errAcceptable := errors.New("any")
|
||||
assert.Equal(t, errAcceptable, b.doReq(func() error {
|
||||
return errAcceptable
|
||||
}, nil, func(err error) bool {
|
||||
return err == errAcceptable
|
||||
}))
|
||||
}
|
||||
|
||||
func TestGoogleBreakerNotAcceptable(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
errAcceptable := errors.New("any")
|
||||
assert.Equal(t, errAcceptable, b.doReq(func() error {
|
||||
return errAcceptable
|
||||
}, nil, func(err error) bool {
|
||||
return err != errAcceptable
|
||||
}))
|
||||
}
|
||||
|
||||
func TestGoogleBreakerPanic(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
assert.Panics(t, func() {
|
||||
_ = b.doReq(func() error {
|
||||
panic("fail")
|
||||
}, nil, defaultAcceptable)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoogleBreakerHalfOpen(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
assert.Nil(t, b.accept())
|
||||
t.Run("accept single failed/accept", func(t *testing.T) {
|
||||
markFailed(b, 10000)
|
||||
time.Sleep(testInterval * 2)
|
||||
verify(t, func() bool {
|
||||
return b.accept() != nil
|
||||
})
|
||||
})
|
||||
t.Run("accept single failed/allow", func(t *testing.T) {
|
||||
markFailed(b, 10000)
|
||||
time.Sleep(testInterval * 2)
|
||||
verify(t, func() bool {
|
||||
_, err := b.allow()
|
||||
return err != nil
|
||||
})
|
||||
})
|
||||
time.Sleep(testInterval * testBuckets)
|
||||
t.Run("accept single succeed", func(t *testing.T) {
|
||||
assert.Nil(t, b.accept())
|
||||
markSuccess(b, 10000)
|
||||
verify(t, func() bool {
|
||||
return b.accept() == nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoogleBreakerSelfProtection(t *testing.T) {
|
||||
t.Run("total request < 100", func(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
markFailed(b, 4)
|
||||
time.Sleep(testInterval)
|
||||
assert.Nil(t, b.accept())
|
||||
})
|
||||
t.Run("total request > 100, total < 2 * success", func(t *testing.T) {
|
||||
b := getGoogleBreaker()
|
||||
size := rand.Intn(10000)
|
||||
accepts := int(math.Ceil(float64(size))) + 1
|
||||
markSuccess(b, accepts)
|
||||
markFailed(b, size-accepts)
|
||||
assert.Nil(t, b.accept())
|
||||
})
|
||||
}
|
||||
|
||||
func TestGoogleBreakerHistory(t *testing.T) {
|
||||
var b *googleBreaker
|
||||
var accepts, total int64
|
||||
|
||||
sleep := testInterval
|
||||
t.Run("accepts == total", func(t *testing.T) {
|
||||
b = getGoogleBreaker()
|
||||
markSuccessWithDuration(b, 10, sleep/2)
|
||||
accepts, total = b.history()
|
||||
assert.Equal(t, int64(10), accepts)
|
||||
assert.Equal(t, int64(10), total)
|
||||
})
|
||||
|
||||
t.Run("fail == total", func(t *testing.T) {
|
||||
b = getGoogleBreaker()
|
||||
markFailedWithDuration(b, 10, sleep/2)
|
||||
accepts, total = b.history()
|
||||
assert.Equal(t, int64(0), accepts)
|
||||
assert.Equal(t, int64(10), total)
|
||||
})
|
||||
|
||||
t.Run("accepts = 1/2 * total, fail = 1/2 * total", func(t *testing.T) {
|
||||
b = getGoogleBreaker()
|
||||
markFailedWithDuration(b, 5, sleep/2)
|
||||
markSuccessWithDuration(b, 5, sleep/2)
|
||||
accepts, total = b.history()
|
||||
assert.Equal(t, int64(5), accepts)
|
||||
assert.Equal(t, int64(10), total)
|
||||
})
|
||||
|
||||
t.Run("auto reset rolling counter", func(t *testing.T) {
|
||||
b = getGoogleBreaker()
|
||||
time.Sleep(testInterval * testBuckets)
|
||||
accepts, total = b.history()
|
||||
assert.Equal(t, int64(0), accepts)
|
||||
assert.Equal(t, int64(0), total)
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkGoogleBreakerAllow(b *testing.B) {
|
||||
breaker := getGoogleBreaker()
|
||||
b.ResetTimer()
|
||||
for i := 0; i <= b.N; i++ {
|
||||
breaker.accept()
|
||||
if i%2 == 0 {
|
||||
breaker.markSuccess()
|
||||
} else {
|
||||
breaker.markFailure()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func markSuccess(b *googleBreaker, count int) {
|
||||
for i := 0; i < count; i++ {
|
||||
p, err := b.allow()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
p.Accept()
|
||||
}
|
||||
}
|
||||
|
||||
func markFailed(b *googleBreaker, count int) {
|
||||
for i := 0; i < count; i++ {
|
||||
p, err := b.allow()
|
||||
if err == nil {
|
||||
p.Reject()
|
||||
}
|
||||
}
|
||||
}
|
||||
42
core/breaker/nopbreaker.go
Normal file
42
core/breaker/nopbreaker.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package breaker
|
||||
|
||||
const noOpBreakerName = "nopBreaker"
|
||||
|
||||
type noOpBreaker struct{}
|
||||
|
||||
func newNoOpBreaker() Breaker {
|
||||
return noOpBreaker{}
|
||||
}
|
||||
|
||||
func (b noOpBreaker) Name() string {
|
||||
return noOpBreakerName
|
||||
}
|
||||
|
||||
func (b noOpBreaker) Allow() (Promise, error) {
|
||||
return nopPromise{}, nil
|
||||
}
|
||||
|
||||
func (b noOpBreaker) Do(req func() error) error {
|
||||
return req()
|
||||
}
|
||||
|
||||
func (b noOpBreaker) DoWithAcceptable(req func() error, acceptable Acceptable) error {
|
||||
return req()
|
||||
}
|
||||
|
||||
func (b noOpBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
|
||||
return req()
|
||||
}
|
||||
|
||||
func (b noOpBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
|
||||
acceptable Acceptable) error {
|
||||
return req()
|
||||
}
|
||||
|
||||
type nopPromise struct{}
|
||||
|
||||
func (p nopPromise) Accept() {
|
||||
}
|
||||
|
||||
func (p nopPromise) Reject(reason string) {
|
||||
}
|
||||
38
core/breaker/nopbreaker_test.go
Normal file
38
core/breaker/nopbreaker_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package breaker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNopBreaker(t *testing.T) {
|
||||
b := newNoOpBreaker()
|
||||
assert.Equal(t, noOpBreakerName, b.Name())
|
||||
p, err := b.Allow()
|
||||
assert.Nil(t, err)
|
||||
p.Accept()
|
||||
for i := 0; i < 1000; i++ {
|
||||
p, err := b.Allow()
|
||||
assert.Nil(t, err)
|
||||
p.Reject("any")
|
||||
}
|
||||
assert.Nil(t, b.Do(func() error {
|
||||
return nil
|
||||
}))
|
||||
assert.Nil(t, b.DoWithAcceptable(func() error {
|
||||
return nil
|
||||
}, defaultAcceptable))
|
||||
errDummy := errors.New("any")
|
||||
assert.Equal(t, errDummy, b.DoWithFallback(func() error {
|
||||
return errDummy
|
||||
}, func(err error) error {
|
||||
return nil
|
||||
}))
|
||||
assert.Equal(t, errDummy, b.DoWithFallbackAcceptable(func() error {
|
||||
return errDummy
|
||||
}, func(err error) error {
|
||||
return nil
|
||||
}, defaultAcceptable))
|
||||
}
|
||||
Reference in New Issue
Block a user