initial import

This commit is contained in:
kevin
2020-07-26 17:09:05 +08:00
commit 7e3a369a8f
647 changed files with 54754 additions and 0 deletions

229
core/breaker/breaker.go Normal file
View File

@@ -0,0 +1,229 @@
package breaker
import (
"errors"
"fmt"
"strings"
"sync"
"time"
"zero/core/mathx"
"zero/core/proc"
"zero/core/stat"
"zero/core/stringx"
)
const (
StateClosed State = iota
StateOpen
)
const (
numHistoryReasons = 5
timeFormat = "15:04:05"
)
// ErrServiceUnavailable is returned when the CB state is open
var ErrServiceUnavailable = errors.New("circuit breaker is open")
type (
State = int32
Acceptable func(err error) bool
Breaker interface {
// Name returns the name of the netflixBreaker.
Name() string
// Allow checks if the request is allowed.
// If allowed, a promise will be returned, the caller needs to call promise.Accept()
// on success, or call promise.Reject() on failure.
// If not allow, ErrServiceUnavailable will be returned.
Allow() (Promise, error)
// Do runs the given request if the netflixBreaker accepts it.
// Do returns an error instantly if the netflixBreaker rejects the request.
// If a panic occurs in the request, the netflixBreaker handles it as an error
// and causes the same panic again.
Do(req func() error) error
// DoWithAcceptable runs the given request if the netflixBreaker accepts it.
// Do returns an error instantly if the netflixBreaker rejects the request.
// If a panic occurs in the request, the netflixBreaker handles it as an error
// and causes the same panic again.
// acceptable checks if it's a successful call, even if the err is not nil.
DoWithAcceptable(req func() error, acceptable Acceptable) error
// DoWithFallback runs the given request if the netflixBreaker accepts it.
// DoWithFallback runs the fallback if the netflixBreaker rejects the request.
// If a panic occurs in the request, the netflixBreaker handles it as an error
// and causes the same panic again.
DoWithFallback(req func() error, fallback func(err error) error) error
// DoWithFallbackAcceptable runs the given request if the netflixBreaker accepts it.
// DoWithFallback runs the fallback if the netflixBreaker rejects the request.
// If a panic occurs in the request, the netflixBreaker handles it as an error
// and causes the same panic again.
// acceptable checks if it's a successful call, even if the err is not nil.
DoWithFallbackAcceptable(req func() error, fallback func(err error) error, acceptable Acceptable) error
}
BreakerOption func(breaker *circuitBreaker)
Promise interface {
Accept()
Reject(reason string)
}
internalPromise interface {
Accept()
Reject()
}
circuitBreaker struct {
name string
throttle
}
internalThrottle interface {
allow() (internalPromise, error)
doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error
}
throttle interface {
allow() (Promise, error)
doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error
}
)
func NewBreaker(opts ...BreakerOption) Breaker {
var b circuitBreaker
for _, opt := range opts {
opt(&b)
}
if len(b.name) == 0 {
b.name = stringx.Rand()
}
b.throttle = newLoggedThrottle(b.name, newGoogleBreaker())
return &b
}
func (cb *circuitBreaker) Allow() (Promise, error) {
return cb.throttle.allow()
}
func (cb *circuitBreaker) Do(req func() error) error {
return cb.throttle.doReq(req, nil, defaultAcceptable)
}
func (cb *circuitBreaker) DoWithAcceptable(req func() error, acceptable Acceptable) error {
return cb.throttle.doReq(req, nil, acceptable)
}
func (cb *circuitBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
return cb.throttle.doReq(req, fallback, defaultAcceptable)
}
func (cb *circuitBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
acceptable Acceptable) error {
return cb.throttle.doReq(req, fallback, acceptable)
}
func (cb *circuitBreaker) Name() string {
return cb.name
}
func WithName(name string) BreakerOption {
return func(b *circuitBreaker) {
b.name = name
}
}
func defaultAcceptable(err error) bool {
return err == nil
}
type loggedThrottle struct {
name string
internalThrottle
errWin *errorWindow
}
func newLoggedThrottle(name string, t internalThrottle) loggedThrottle {
return loggedThrottle{
name: name,
internalThrottle: t,
errWin: new(errorWindow),
}
}
func (lt loggedThrottle) allow() (Promise, error) {
promise, err := lt.internalThrottle.allow()
return promiseWithReason{
promise: promise,
errWin: lt.errWin,
}, lt.logError(err)
}
func (lt loggedThrottle) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
return lt.logError(lt.internalThrottle.doReq(req, fallback, func(err error) bool {
accept := acceptable(err)
if !accept {
lt.errWin.add(err.Error())
}
return accept
}))
}
func (lt loggedThrottle) logError(err error) error {
if err == ErrServiceUnavailable {
// if circuit open, not possible to have empty error window
stat.Report(fmt.Sprintf(
"proc(%s/%d), callee: %s, breaker is open and requests dropped\nlast errors:\n%s",
proc.ProcessName(), proc.Pid(), lt.name, lt.errWin))
}
return err
}
type errorWindow struct {
reasons [numHistoryReasons]string
index int
count int
lock sync.Mutex
}
func (ew *errorWindow) add(reason string) {
ew.lock.Lock()
ew.reasons[ew.index] = fmt.Sprintf("%s %s", time.Now().Format(timeFormat), reason)
ew.index = (ew.index + 1) % numHistoryReasons
ew.count = mathx.MinInt(ew.count+1, numHistoryReasons)
ew.lock.Unlock()
}
func (ew *errorWindow) String() string {
var builder strings.Builder
ew.lock.Lock()
for i := ew.index + ew.count - 1; i >= ew.index; i-- {
builder.WriteString(ew.reasons[i%numHistoryReasons])
builder.WriteByte('\n')
}
ew.lock.Unlock()
return builder.String()
}
type promiseWithReason struct {
promise internalPromise
errWin *errorWindow
}
func (p promiseWithReason) Accept() {
p.promise.Accept()
}
func (p promiseWithReason) Reject(reason string) {
p.errWin.add(reason)
p.promise.Reject()
}

View File

@@ -0,0 +1,44 @@
package breaker
import (
"errors"
"strconv"
"testing"
"zero/core/stat"
"github.com/stretchr/testify/assert"
)
func init() {
stat.SetReporter(nil)
}
func TestCircuitBreaker_Allow(t *testing.T) {
b := NewBreaker()
assert.True(t, len(b.Name()) > 0)
_, err := b.Allow()
assert.Nil(t, err)
}
func TestLogReason(t *testing.T) {
b := NewBreaker()
assert.True(t, len(b.Name()) > 0)
for i := 0; i < 1000; i++ {
_ = b.Do(func() error {
return errors.New(strconv.Itoa(i))
})
}
errs := b.(*circuitBreaker).throttle.(loggedThrottle).errWin
assert.Equal(t, numHistoryReasons, errs.count)
}
func BenchmarkGoogleBreaker(b *testing.B) {
br := NewBreaker()
for i := 0; i < b.N; i++ {
_ = br.Do(func() error {
return nil
})
}
}

76
core/breaker/breakers.go Normal file
View File

@@ -0,0 +1,76 @@
package breaker
import "sync"
var (
lock sync.RWMutex
breakers = make(map[string]Breaker)
)
func Do(name string, req func() error) error {
return do(name, func(b Breaker) error {
return b.Do(req)
})
}
func DoWithAcceptable(name string, req func() error, acceptable Acceptable) error {
return do(name, func(b Breaker) error {
return b.DoWithAcceptable(req, acceptable)
})
}
func DoWithFallback(name string, req func() error, fallback func(err error) error) error {
return do(name, func(b Breaker) error {
return b.DoWithFallback(req, fallback)
})
}
func DoWithFallbackAcceptable(name string, req func() error, fallback func(err error) error,
acceptable Acceptable) error {
return do(name, func(b Breaker) error {
return b.DoWithFallbackAcceptable(req, fallback, acceptable)
})
}
func GetBreaker(name string) Breaker {
lock.RLock()
b, ok := breakers[name]
lock.RUnlock()
if ok {
return b
}
lock.Lock()
defer lock.Unlock()
b = NewBreaker()
breakers[name] = b
return b
}
func NoBreakFor(name string) {
lock.Lock()
breakers[name] = newNoOpBreaker()
lock.Unlock()
}
func do(name string, execute func(b Breaker) error) error {
lock.RLock()
b, ok := breakers[name]
lock.RUnlock()
if ok {
return execute(b)
} else {
lock.Lock()
b, ok = breakers[name]
if ok {
lock.Unlock()
return execute(b)
} else {
b = NewBreaker(WithName(name))
breakers[name] = b
lock.Unlock()
return execute(b)
}
}
}

View File

@@ -0,0 +1,115 @@
package breaker
import (
"errors"
"fmt"
"testing"
"zero/core/stat"
"github.com/stretchr/testify/assert"
)
func init() {
stat.SetReporter(nil)
}
func TestBreakersDo(t *testing.T) {
assert.Nil(t, Do("any", func() error {
return nil
}))
errDummy := errors.New("any")
assert.Equal(t, errDummy, Do("any", func() error {
return errDummy
}))
}
func TestBreakersDoWithAcceptable(t *testing.T) {
errDummy := errors.New("anyone")
for i := 0; i < 10000; i++ {
assert.Equal(t, errDummy, GetBreaker("anyone").DoWithAcceptable(func() error {
return errDummy
}, func(err error) bool {
return err == nil || err == errDummy
}))
}
verify(t, func() bool {
return Do("anyone", func() error {
return nil
}) == nil
})
for i := 0; i < 10000; i++ {
err := DoWithAcceptable("another", func() error {
return errDummy
}, func(err error) bool {
return err == nil
})
assert.True(t, err == errDummy || err == ErrServiceUnavailable)
}
verify(t, func() bool {
return ErrServiceUnavailable == Do("another", func() error {
return nil
})
})
}
func TestBreakersNoBreakerFor(t *testing.T) {
NoBreakFor("any")
errDummy := errors.New("any")
for i := 0; i < 10000; i++ {
assert.Equal(t, errDummy, GetBreaker("any").Do(func() error {
return errDummy
}))
}
assert.Equal(t, nil, Do("any", func() error {
return nil
}))
}
func TestBreakersFallback(t *testing.T) {
errDummy := errors.New("any")
for i := 0; i < 10000; i++ {
err := DoWithFallback("fallback", func() error {
return errDummy
}, func(err error) error {
return nil
})
assert.True(t, err == nil || err == errDummy)
}
verify(t, func() bool {
return ErrServiceUnavailable == Do("fallback", func() error {
return nil
})
})
}
func TestBreakersAcceptableFallback(t *testing.T) {
errDummy := errors.New("any")
for i := 0; i < 10000; i++ {
err := DoWithFallbackAcceptable("acceptablefallback", func() error {
return errDummy
}, func(err error) error {
return nil
}, func(err error) bool {
return err == nil
})
assert.True(t, err == nil || err == errDummy)
}
verify(t, func() bool {
return ErrServiceUnavailable == Do("acceptablefallback", func() error {
return nil
})
})
}
func verify(t *testing.T, fn func() bool) {
var count int
for i := 0; i < 100; i++ {
if fn() {
count++
}
}
assert.True(t, count >= 80, fmt.Sprintf("should be greater than 80, actual %d", count))
}

View File

@@ -0,0 +1,125 @@
package breaker
import (
"math"
"sync/atomic"
"time"
"zero/core/collection"
"zero/core/mathx"
)
const (
// 250ms for bucket duration
window = time.Second * 10
buckets = 40
k = 1.5
protection = 5
)
// googleBreaker is a netflixBreaker pattern from google.
// see Client-Side Throttling section in https://landing.google.com/sre/sre-book/chapters/handling-overload/
type googleBreaker struct {
k float64
state int32
stat *collection.RollingWindow
proba *mathx.Proba
}
func newGoogleBreaker() *googleBreaker {
bucketDuration := time.Duration(int64(window) / int64(buckets))
st := collection.NewRollingWindow(buckets, bucketDuration)
return &googleBreaker{
stat: st,
k: k,
state: StateClosed,
proba: mathx.NewProba(),
}
}
func (b *googleBreaker) accept() error {
accepts, total := b.history()
weightedAccepts := b.k * float64(accepts)
// https://landing.google.com/sre/sre-book/chapters/handling-overload/#eq2101
dropRatio := math.Max(0, (float64(total-protection)-weightedAccepts)/float64(total+1))
if dropRatio <= 0 {
if atomic.LoadInt32(&b.state) == StateOpen {
atomic.CompareAndSwapInt32(&b.state, StateOpen, StateClosed)
}
return nil
}
if atomic.LoadInt32(&b.state) == StateClosed {
atomic.CompareAndSwapInt32(&b.state, StateClosed, StateOpen)
}
if b.proba.TrueOnProba(dropRatio) {
return ErrServiceUnavailable
}
return nil
}
func (b *googleBreaker) allow() (internalPromise, error) {
if err := b.accept(); err != nil {
return nil, err
}
return googlePromise{
b: b,
}, nil
}
func (b *googleBreaker) doReq(req func() error, fallback func(err error) error, acceptable Acceptable) error {
if err := b.accept(); err != nil {
if fallback != nil {
return fallback(err)
} else {
return err
}
}
defer func() {
if e := recover(); e != nil {
b.markFailure()
panic(e)
}
}()
err := req()
if acceptable(err) {
b.markSuccess()
} else {
b.markFailure()
}
return err
}
func (b *googleBreaker) markSuccess() {
b.stat.Add(1)
}
func (b *googleBreaker) markFailure() {
b.stat.Add(0)
}
func (b *googleBreaker) history() (accepts int64, total int64) {
b.stat.Reduce(func(b *collection.Bucket) {
accepts += int64(b.Sum)
total += b.Count
})
return
}
type googlePromise struct {
b *googleBreaker
}
func (p googlePromise) Accept() {
p.b.markSuccess()
}
func (p googlePromise) Reject() {
p.b.markFailure()
}

View File

@@ -0,0 +1,238 @@
package breaker
import (
"errors"
"math"
"math/rand"
"testing"
"time"
"zero/core/collection"
"zero/core/mathx"
"zero/core/stat"
"github.com/stretchr/testify/assert"
)
const (
testBuckets = 10
testInterval = time.Millisecond * 10
)
func init() {
stat.SetReporter(nil)
}
func getGoogleBreaker() *googleBreaker {
st := collection.NewRollingWindow(testBuckets, testInterval)
return &googleBreaker{
stat: st,
k: 5,
state: StateClosed,
proba: mathx.NewProba(),
}
}
func markSuccessWithDuration(b *googleBreaker, count int, sleep time.Duration) {
for i := 0; i < count; i++ {
b.markSuccess()
time.Sleep(sleep)
}
}
func markFailedWithDuration(b *googleBreaker, count int, sleep time.Duration) {
for i := 0; i < count; i++ {
b.markFailure()
time.Sleep(sleep)
}
}
func TestGoogleBreakerClose(t *testing.T) {
b := getGoogleBreaker()
markSuccess(b, 80)
assert.Nil(t, b.accept())
markSuccess(b, 120)
assert.Nil(t, b.accept())
}
func TestGoogleBreakerOpen(t *testing.T) {
b := getGoogleBreaker()
markSuccess(b, 10)
assert.Nil(t, b.accept())
markFailed(b, 100000)
time.Sleep(testInterval * 2)
verify(t, func() bool {
return b.accept() != nil
})
}
func TestGoogleBreakerFallback(t *testing.T) {
b := getGoogleBreaker()
markSuccess(b, 1)
assert.Nil(t, b.accept())
markFailed(b, 10000)
time.Sleep(testInterval * 2)
verify(t, func() bool {
return b.doReq(func() error {
return errors.New("any")
}, func(err error) error {
return nil
}, defaultAcceptable) == nil
})
}
func TestGoogleBreakerReject(t *testing.T) {
b := getGoogleBreaker()
markSuccess(b, 100)
assert.Nil(t, b.accept())
markFailed(b, 10000)
time.Sleep(testInterval)
assert.Equal(t, ErrServiceUnavailable, b.doReq(func() error {
return ErrServiceUnavailable
}, nil, defaultAcceptable))
}
func TestGoogleBreakerAcceptable(t *testing.T) {
b := getGoogleBreaker()
errAcceptable := errors.New("any")
assert.Equal(t, errAcceptable, b.doReq(func() error {
return errAcceptable
}, nil, func(err error) bool {
return err == errAcceptable
}))
}
func TestGoogleBreakerNotAcceptable(t *testing.T) {
b := getGoogleBreaker()
errAcceptable := errors.New("any")
assert.Equal(t, errAcceptable, b.doReq(func() error {
return errAcceptable
}, nil, func(err error) bool {
return err != errAcceptable
}))
}
func TestGoogleBreakerPanic(t *testing.T) {
b := getGoogleBreaker()
assert.Panics(t, func() {
_ = b.doReq(func() error {
panic("fail")
}, nil, defaultAcceptable)
})
}
func TestGoogleBreakerHalfOpen(t *testing.T) {
b := getGoogleBreaker()
assert.Nil(t, b.accept())
t.Run("accept single failed/accept", func(t *testing.T) {
markFailed(b, 10000)
time.Sleep(testInterval * 2)
verify(t, func() bool {
return b.accept() != nil
})
})
t.Run("accept single failed/allow", func(t *testing.T) {
markFailed(b, 10000)
time.Sleep(testInterval * 2)
verify(t, func() bool {
_, err := b.allow()
return err != nil
})
})
time.Sleep(testInterval * testBuckets)
t.Run("accept single succeed", func(t *testing.T) {
assert.Nil(t, b.accept())
markSuccess(b, 10000)
verify(t, func() bool {
return b.accept() == nil
})
})
}
func TestGoogleBreakerSelfProtection(t *testing.T) {
t.Run("total request < 100", func(t *testing.T) {
b := getGoogleBreaker()
markFailed(b, 4)
time.Sleep(testInterval)
assert.Nil(t, b.accept())
})
t.Run("total request > 100, total < 2 * success", func(t *testing.T) {
b := getGoogleBreaker()
size := rand.Intn(10000)
accepts := int(math.Ceil(float64(size))) + 1
markSuccess(b, accepts)
markFailed(b, size-accepts)
assert.Nil(t, b.accept())
})
}
func TestGoogleBreakerHistory(t *testing.T) {
var b *googleBreaker
var accepts, total int64
sleep := testInterval
t.Run("accepts == total", func(t *testing.T) {
b = getGoogleBreaker()
markSuccessWithDuration(b, 10, sleep/2)
accepts, total = b.history()
assert.Equal(t, int64(10), accepts)
assert.Equal(t, int64(10), total)
})
t.Run("fail == total", func(t *testing.T) {
b = getGoogleBreaker()
markFailedWithDuration(b, 10, sleep/2)
accepts, total = b.history()
assert.Equal(t, int64(0), accepts)
assert.Equal(t, int64(10), total)
})
t.Run("accepts = 1/2 * total, fail = 1/2 * total", func(t *testing.T) {
b = getGoogleBreaker()
markFailedWithDuration(b, 5, sleep/2)
markSuccessWithDuration(b, 5, sleep/2)
accepts, total = b.history()
assert.Equal(t, int64(5), accepts)
assert.Equal(t, int64(10), total)
})
t.Run("auto reset rolling counter", func(t *testing.T) {
b = getGoogleBreaker()
time.Sleep(testInterval * testBuckets)
accepts, total = b.history()
assert.Equal(t, int64(0), accepts)
assert.Equal(t, int64(0), total)
})
}
func BenchmarkGoogleBreakerAllow(b *testing.B) {
breaker := getGoogleBreaker()
b.ResetTimer()
for i := 0; i <= b.N; i++ {
breaker.accept()
if i%2 == 0 {
breaker.markSuccess()
} else {
breaker.markFailure()
}
}
}
func markSuccess(b *googleBreaker, count int) {
for i := 0; i < count; i++ {
p, err := b.allow()
if err != nil {
break
}
p.Accept()
}
}
func markFailed(b *googleBreaker, count int) {
for i := 0; i < count; i++ {
p, err := b.allow()
if err == nil {
p.Reject()
}
}
}

View File

@@ -0,0 +1,42 @@
package breaker
const noOpBreakerName = "nopBreaker"
type noOpBreaker struct{}
func newNoOpBreaker() Breaker {
return noOpBreaker{}
}
func (b noOpBreaker) Name() string {
return noOpBreakerName
}
func (b noOpBreaker) Allow() (Promise, error) {
return nopPromise{}, nil
}
func (b noOpBreaker) Do(req func() error) error {
return req()
}
func (b noOpBreaker) DoWithAcceptable(req func() error, acceptable Acceptable) error {
return req()
}
func (b noOpBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
return req()
}
func (b noOpBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
acceptable Acceptable) error {
return req()
}
type nopPromise struct{}
func (p nopPromise) Accept() {
}
func (p nopPromise) Reject(reason string) {
}

View File

@@ -0,0 +1,38 @@
package breaker
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
)
func TestNopBreaker(t *testing.T) {
b := newNoOpBreaker()
assert.Equal(t, noOpBreakerName, b.Name())
p, err := b.Allow()
assert.Nil(t, err)
p.Accept()
for i := 0; i < 1000; i++ {
p, err := b.Allow()
assert.Nil(t, err)
p.Reject("any")
}
assert.Nil(t, b.Do(func() error {
return nil
}))
assert.Nil(t, b.DoWithAcceptable(func() error {
return nil
}, defaultAcceptable))
errDummy := errors.New("any")
assert.Equal(t, errDummy, b.DoWithFallback(func() error {
return errDummy
}, func(err error) error {
return nil
}))
assert.Equal(t, errDummy, b.DoWithFallbackAcceptable(func() error {
return errDummy
}, func(err error) error {
return nil
}, defaultAcceptable))
}