Add grpc retry (#1160)

* Add grpc retry

* Update grpc retry

* Add tests

* Fix a bug

* Add api && some tests

* Add comment

* Add double check

* Add server retry quota

* Update optimize code

* Fix bug

* Update optimize code

* Update optimize code

* Fix bug

core/retry/backoff/backoff.go

@@ -0,0 +1,31 @@
+package backoff
+
+import (
+	"math/rand"
+	"time"
+)
+
+type Func func(attempt int) time.Duration
+
+// LinearWithJitter waits a set period of time, allowing for jitter (fractional adjustment).
+func LinearWithJitter(waitBetween time.Duration, jitterFraction float64) Func {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	return func(attempt int) time.Duration {
+		multiplier := jitterFraction * (r.Float64()*2 - 1)
+		return time.Duration(float64(waitBetween) * (1 + multiplier))
+	}
+}
+
+// Interval it waits for a fixed period of time between calls.
+func Interval(interval time.Duration) Func {
+	return func(attempt int) time.Duration {
+		return interval
+	}
+}
+
+// Exponential produces increasing intervals for each attempt.
+func Exponential(scalar time.Duration) Func {
+	return func(attempt int) time.Duration {
+		return scalar * time.Duration((1<<attempt)>>1)
+	}
+}

core/retry/backoff/backoff_test.go

@@ -0,0 +1,17 @@
+package backoff
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestWaitBetween(t *testing.T) {
+	fn := Interval(time.Second)
+	assert.EqualValues(t, time.Second, fn(1))
+}
+
+func TestExponential(t *testing.T) {
+	fn := Exponential(time.Second)
+	assert.EqualValues(t, time.Second, fn(1))
+}