zeromicro/go-zero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feature(retry): Delete retry mechanism (#1279)

Browse Source
This commit is contained in:
chenquan
2021-11-27 11:32:33 +08:00
committed by GitHub
parent de5ed6a677
commit cf683411ee
18 changed files with 4 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
core/retry/backoff/backoff.go
View File

@@ -1,32 +0,0 @@
package backoff
import (
"math/rand"
"time"
)
// Func defines the method to calculate how long to retry.
type Func func(attempt int) time.Duration
// LinearWithJitter waits a set period of time, allowing for jitter (fractional adjustment).
func LinearWithJitter(waitBetween time.Duration, jitterFraction float64) Func {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return func(attempt int) time.Duration {
multiplier := jitterFraction * (r.Float64()*2 - 1)
return time.Duration(float64(waitBetween) * (1 + multiplier))
}
}
// Interval it waits for a fixed period of time between calls.
func Interval(interval time.Duration) Func {
return func(attempt int) time.Duration {
return interval
}
}
// Exponential produces increasing intervals for each attempt.
func Exponential(scalar time.Duration) Func {
return func(attempt int) time.Duration {
return scalar * time.Duration((1<<attempt)>>1)
}
}

30
core/retry/backoff/backoff_test.go
View File

@@ -1,30 +0,0 @@
package backoff
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestWaitBetween(t *testing.T) {
fn := Interval(time.Second)
assert.EqualValues(t, time.Second, fn(1))
}
func TestExponential(t *testing.T) {
fn := Exponential(time.Second)
assert.EqualValues(t, time.Second, fn(1))
}
func TestLinearWithJitter(t *testing.T) {
const rounds = 1000000
var total time.Duration
fn := LinearWithJitter(time.Second, 0.5)
for i := 0; i < rounds; i++ {
total += fn(1)
}
// 0.1% tolerance
assert.True(t, total/time.Duration(rounds)-time.Second < time.Millisecond)
}

42
core/retry/options.go
View File

@@ -1,42 +0,0 @@
package retry
import (
"time"
"github.com/tal-tech/go-zero/core/retry/backoff"
"google.golang.org/grpc/codes"
)
// WithDisable disables the retry behaviour on this call, or this interceptor.
// It's semantically the same to `WithMax(0)`
func WithDisable() *CallOption {
return WithMax(0)
}
// WithMax sets the maximum number of retries on this call, or this interceptor.
func WithMax(maxRetries int) *CallOption {
return &CallOption{apply: func(options *options) {
options.max = maxRetries
}}
}
// WithBackoff sets the `BackoffFunc` used to control time between retries.
func WithBackoff(backoffFunc backoff.Func) *CallOption {
return &CallOption{apply: func(o *options) {
o.backoffFunc = backoffFunc
}}
}
// WithCodes Allow code to be retried.
func WithCodes(retryCodes ...codes.Code) *CallOption {
return &CallOption{apply: func(o *options) {
o.codes = retryCodes
}}
}
// WithPerRetryTimeout timeout for each retry
func WithPerRetryTimeout(timeout time.Duration) *CallOption {
return &CallOption{apply: func(o *options) {
o.perCallTimeout = timeout
}}
}

91
core/retry/options_test.go
View File

@@ -1,91 +0,0 @@
package retry
import (
"context"
"errors"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/tal-tech/go-zero/core/logx"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
func TestRetryWithDisable(t *testing.T) {
opt := &options{}
assert.EqualValues(t, &options{}, parseRetryCallOptions(opt, WithDisable()))
}
func TestRetryWithMax(t *testing.T) {
n := 5
for i := 0; i < n; i++ {
opt := &options{}
assert.EqualValues(t, &options{max: i}, parseRetryCallOptions(opt, WithMax(i)))
}
}
func TestRetryWithBackoff(t *testing.T) {
opt := &options{}
retryCallOptions := parseRetryCallOptions(opt, WithBackoff(func(attempt int) time.Duration {
return time.Millisecond
}))
assert.EqualValues(t, time.Millisecond, retryCallOptions.backoffFunc(1))
}
func TestRetryWithCodes(t *testing.T) {
opt := &options{}
c := []codes.Code{codes.Unknown, codes.NotFound}
options := parseRetryCallOptions(opt, WithCodes(c...))
assert.EqualValues(t, c, options.codes)
}
func TestRetryWithPerRetryTimeout(t *testing.T) {
opt := &options{}
options := parseRetryCallOptions(opt, WithPerRetryTimeout(time.Millisecond))
assert.EqualValues(t, time.Millisecond, options.perCallTimeout)
}
func Test_waitRetryBackoff(t *testing.T) {
logx.Disable()
opt := &options{perCallTimeout: time.Second, backoffFunc: func(attempt int) time.Duration {
return time.Second
}}
logger := logx.WithContext(context.Background())
err := waitRetryBackoff(logger, 1, context.Background(), opt)
assert.NoError(t, err)
ctx, cancelFunc := context.WithTimeout(context.Background(), time.Millisecond)
defer cancelFunc()
err = waitRetryBackoff(logger, 1, ctx, opt)
assert.ErrorIs(t, err, status.FromContextError(context.DeadlineExceeded).Err())
}
func Test_isRetriable(t *testing.T) {
assert.False(t, isRetriable(status.FromContextError(context.DeadlineExceeded).Err(), &options{codes: DefaultRetriableCodes}))
assert.True(t, isRetriable(status.Error(codes.ResourceExhausted, ""), &options{codes: DefaultRetriableCodes}))
assert.False(t, isRetriable(errors.New("error"), &options{}))
}
func Test_perCallContext(t *testing.T) {
opt := &options{perCallTimeout: time.Second, includeRetryHeader: true}
ctx := metadata.NewIncomingContext(context.Background(), map[string][]string{"1": {"1"}})
callContext := perCallContext(ctx, opt, 1)
md, ok := metadata.FromOutgoingContext(callContext)
assert.True(t, ok)
assert.EqualValues(t, metadata.MD{"1": {"1"}, AttemptMetadataKey: {"1"}}, md)
}
func Test_filterCallOptions(t *testing.T) {
grpcEmptyCallOpt := &grpc.EmptyCallOption{}
retryCallOpt := &CallOption{}
options, retryCallOptions := filterCallOptions([]grpc.CallOption{
grpcEmptyCallOpt,
retryCallOpt,
})
assert.EqualValues(t, []grpc.CallOption{grpcEmptyCallOpt}, options)
assert.EqualValues(t, []*CallOption{retryCallOpt}, retryCallOptions)
}

189
core/retry/retryinterceptor.go
View File

@@ -1,189 +0,0 @@
package retry
import (
"context"
"strconv"
"time"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/retry/backoff"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
const AttemptMetadataKey = "x-retry-attempt"
var (
// DefaultRetriableCodes default retry code
DefaultRetriableCodes = []codes.Code{codes.ResourceExhausted, codes.Unavailable}
// defaultRetryOptions default retry configuration
defaultRetryOptions = &options{
max: 0, // disabled
perCallTimeout: 0, // disabled
includeRetryHeader: true,
codes: DefaultRetriableCodes,
backoffFunc: backoff.LinearWithJitter(50*time.Millisecond /*jitter*/, 0.10),
}
)
type (
// options retry the configuration
options struct {
max int
perCallTimeout time.Duration
includeRetryHeader bool
codes []codes.Code
backoffFunc backoff.Func
}
// CallOption is a grpc.CallOption that is local to grpc retry.
CallOption struct {
grpc.EmptyCallOption // make sure we implement private after() and before() fields so we don't panic.
apply func(opt *options)
}
)
func waitRetryBackoff(logger logx.Logger, attempt int, ctx context.Context, retryOptions *options) error {
var waitTime time.Duration = 0
if attempt > 0 {
waitTime = retryOptions.backoffFunc(attempt)
}
if waitTime > 0 {
timer := time.NewTimer(waitTime)
defer timer.Stop()
logger.Infof("grpc retry attempt: %d, backoff for %v", attempt, waitTime)
select {
case <-ctx.Done():
return status.FromContextError(ctx.Err()).Err()
case <-timer.C:
// double check
err := ctx.Err()
if err != nil {
return status.FromContextError(err).Err()
}
}
}
return nil
}
func isRetriable(err error, retryOptions *options) bool {
errCode := status.Code(err)
if isContextError(err) {
return false
}
for _, code := range retryOptions.codes {
if code == errCode {
return true
}
}
return false
}
func isContextError(err error) bool {
code := status.Code(err)
return code == codes.DeadlineExceeded || code == codes.Canceled
}
func reuseOrNewWithCallOptions(opt *options, retryCallOptions []*CallOption) *options {
if len(retryCallOptions) == 0 {
return opt
}
return parseRetryCallOptions(opt, retryCallOptions...)
}
func parseRetryCallOptions(opt *options, opts ...*CallOption) *options {
for _, option := range opts {
option.apply(opt)
}
return opt
}
func perCallContext(ctx context.Context, callOpts *options, attempt int) context.Context {
if attempt > 0 {
if callOpts.perCallTimeout != 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, callOpts.perCallTimeout)
_ = cancel
}
if callOpts.includeRetryHeader {
cloneMd := extractIncomingAndClone(ctx)
cloneMd.Set(AttemptMetadataKey, strconv.Itoa(attempt))
ctx = metadata.NewOutgoingContext(ctx, cloneMd)
}
}
return ctx
}
func extractIncomingAndClone(ctx context.Context) metadata.MD {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return metadata.MD{}
}
return md.Copy()
}
func filterCallOptions(callOptions []grpc.CallOption) (grpcOptions []grpc.CallOption, retryOptions []*CallOption) {
for _, opt := range callOptions {
if co, ok := opt.(*CallOption); ok {
retryOptions = append(retryOptions, co)
} else {
grpcOptions = append(grpcOptions, opt)
}
}
return grpcOptions, retryOptions
}
func Do(ctx context.Context, call func(ctx context.Context, opts ...grpc.CallOption) error, opts ...grpc.CallOption) error {
logger := logx.WithContext(ctx)
grpcOpts, retryOpts := filterCallOptions(opts)
callOpts := reuseOrNewWithCallOptions(defaultRetryOptions, retryOpts)
if callOpts.max == 0 {
return call(ctx, opts...)
}
var lastErr error
for attempt := 0; attempt <= callOpts.max; attempt++ {
if err := waitRetryBackoff(logger, attempt, ctx, callOpts); err != nil {
return err
}
callCtx := perCallContext(ctx, callOpts, attempt)
lastErr = call(callCtx, grpcOpts...)
if lastErr == nil {
return nil
}
if attempt == 0 {
logger.Errorf("grpc call failed, got err: %v", lastErr)
} else {
logger.Errorf("grpc retry attempt: %d, got err: %v", attempt, lastErr)
}
if isContextError(lastErr) {
if ctx.Err() != nil {
logger.Errorf("grpc retry attempt: %d, parent context error: %v", attempt, ctx.Err())
return lastErr
} else if callOpts.perCallTimeout != 0 {
logger.Errorf("grpc retry attempt: %d, context error from retry call", attempt)
continue
}
}
if !isRetriable(lastErr, callOpts) {
return lastErr
}
}
return lastErr
}

24
core/retry/retryinterceptor_test.go
View File

@@ -1,24 +0,0 @@
package retry
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func TestDo(t *testing.T) {
n := 4
for i := 0; i < n; i++ {
count := 0
err := Do(context.Background(), func(ctx context.Context, opts ...grpc.CallOption) error {
count++
return status.Error(codes.ResourceExhausted, "ResourceExhausted")
}, WithMax(i))
assert.Error(t, err)
assert.Equal(t, i+1, count)
}
}

6
zrpc/client.go
View File

@@ -18,8 +18,6 @@ var (
WithNonBlock = internal.WithNonBlock WithNonBlock = internal.WithNonBlock
// WithTimeout is an alias of internal.WithTimeout. // WithTimeout is an alias of internal.WithTimeout.
WithTimeout = internal.WithTimeout WithTimeout = internal.WithTimeout
// WithRetry is an alias of internal.WithRetry.
WithRetry = internal.WithRetry
// WithTransportCredentials return a func to make the gRPC calls secured with given credentials. // WithTransportCredentials return a func to make the gRPC calls secured with given credentials.
WithTransportCredentials = internal.WithTransportCredentials WithTransportCredentials = internal.WithTransportCredentials
// WithUnaryClientInterceptor is an alias of internal.WithUnaryClientInterceptor. // WithUnaryClientInterceptor is an alias of internal.WithUnaryClientInterceptor.
@@ -63,9 +61,7 @@ func NewClient(c RpcClientConf, options ...ClientOption) (Client, error) {
if c.Timeout > 0 { if c.Timeout > 0 {
opts = append(opts, WithTimeout(time.Duration(c.Timeout)*time.Millisecond)) opts = append(opts, WithTimeout(time.Duration(c.Timeout)*time.Millisecond))
} }
if c.Retry {
opts = append(opts, WithRetry())
}
opts = append(opts, options...) opts = append(opts, options...)
var target string var target string

1
zrpc/client_test.go
View File

@@ -102,7 +102,6 @@ func TestDepositServer_Deposit(t *testing.T) {
App: "foo", App: "foo",
Token: "bar", Token: "bar",
Timeout: 1000, Timeout: 1000,
Retry: true,
}, },
WithDialOption(grpc.WithContextDialer(dialer())), WithDialOption(grpc.WithContextDialer(dialer())),
WithUnaryClientInterceptor(func(ctx context.Context, method string, req, reply interface{}, WithUnaryClientInterceptor(func(ctx context.Context, method string, req, reply interface{},

2
zrpc/config.go
View File

@@ -18,7 +18,6 @@ type (
// setting 0 means no timeout // setting 0 means no timeout
Timeout int64 `json:",default=2000"` Timeout int64 `json:",default=2000"`
CpuThreshold int64 `json:",default=900,range=[0:1000]"` CpuThreshold int64 `json:",default=900,range=[0:1000]"`
MaxRetries int `json:",default=0,range=[0:]"`
} }
// A RpcClientConf is a rpc client config. // A RpcClientConf is a rpc client config.
@@ -29,7 +28,6 @@ type (
App string `json:",optional"` App string `json:",optional"`
Token string `json:",optional"` Token string `json:",optional"`
NonBlock bool `json:",optional"` NonBlock bool `json:",optional"`
Retry bool `json:",optional"` // grpc auto retry
Timeout int64 `json:",default=2000"` Timeout int64 `json:",default=2000"`
} }
) )

9
zrpc/internal/client.go
View File

@@ -34,7 +34,6 @@ type (
NonBlock bool NonBlock bool
Timeout time.Duration Timeout time.Duration
Secure bool Secure bool
Retry bool
DialOptions []grpc.DialOption DialOptions []grpc.DialOption
} }
@@ -83,7 +82,6 @@ func (c *client) buildDialOptions(opts ...ClientOption) []grpc.DialOption {
clientinterceptors.PrometheusInterceptor, clientinterceptors.PrometheusInterceptor,
clientinterceptors.BreakerInterceptor, clientinterceptors.BreakerInterceptor,
clientinterceptors.TimeoutInterceptor(cliOpts.Timeout), clientinterceptors.TimeoutInterceptor(cliOpts.Timeout),
clientinterceptors.RetryInterceptor(cliOpts.Retry),
), ),
WithStreamClientInterceptors( WithStreamClientInterceptors(
clientinterceptors.StreamTracingInterceptor, clientinterceptors.StreamTracingInterceptor,
@@ -136,13 +134,6 @@ func WithTimeout(timeout time.Duration) ClientOption {
} }
} }
// WithRetry returns a func to customize a ClientOptions with auto retry.
func WithRetry() ClientOption {
return func(options *ClientOptions) {
options.Retry = true
}
}
// WithTransportCredentials return a func to make the gRPC calls secured with given credentials. // WithTransportCredentials return a func to make the gRPC calls secured with given credentials.
func WithTransportCredentials(creds credentials.TransportCredentials) ClientOption { func WithTransportCredentials(creds credentials.TransportCredentials) ClientOption {
return func(options *ClientOptions) { return func(options *ClientOptions) {

7
zrpc/internal/client_test.go
View File

@@ -24,13 +24,6 @@ func TestWithTimeout(t *testing.T) {
assert.Equal(t, time.Second, options.Timeout) assert.Equal(t, time.Second, options.Timeout)
} }
func TestWithRetry(t *testing.T) {
var options ClientOptions
opt := WithRetry()
opt(&options)
assert.True(t, options.Retry)
}
func TestWithNonBlock(t *testing.T) { func TestWithNonBlock(t *testing.T) {
var options ClientOptions var options ClientOptions
opt := WithNonBlock() opt := WithNonBlock()

25
zrpc/internal/clientinterceptors/retryinterceptor.go
View File

@@ -1,25 +0,0 @@
package clientinterceptors
import (
"context"
"github.com/tal-tech/go-zero/core/retry"
"google.golang.org/grpc"
)
// RetryInterceptor retry interceptor
func RetryInterceptor(enable bool) grpc.UnaryClientInterceptor {
if !enable {
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(ctx, method, req, reply, cc, opts...)
}
}
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return retry.Do(ctx, func(ctx context.Context, callOpts ...grpc.CallOption) error {
return invoker(ctx, method, req, reply, cc, callOpts...)
}, opts...)
}
}

28
zrpc/internal/clientinterceptors/retryinterceptor_test.go
View File

@@ -1,28 +0,0 @@
package clientinterceptors
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/tal-tech/go-zero/core/retry"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func TestRetryInterceptor_WithMax(t *testing.T) {
n := 4
for i := 0; i < n; i++ {
count := 0
cc := new(grpc.ClientConn)
err := RetryInterceptor(true)(context.Background(), "/1", nil, nil, cc,
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
opts ...grpc.CallOption) error {
count++
return status.Error(codes.ResourceExhausted, "ResourceExhausted")
}, retry.WithMax(i))
assert.Error(t, err)
assert.Equal(t, i+1, count)
}
}

11
zrpc/internal/rpcserver.go
View File

@@ -14,8 +14,7 @@ type (
ServerOption func(options *rpcServerOptions) ServerOption func(options *rpcServerOptions)
rpcServerOptions struct { rpcServerOptions struct {
metrics *stat.Metrics metrics *stat.Metrics
MaxRetries int
} }
rpcServer struct { rpcServer struct {
@@ -56,7 +55,6 @@ func (s *rpcServer) Start(register RegisterFn) error {
unaryInterceptors := []grpc.UnaryServerInterceptor{ unaryInterceptors := []grpc.UnaryServerInterceptor{
serverinterceptors.UnaryTracingInterceptor, serverinterceptors.UnaryTracingInterceptor,
serverinterceptors.RetryInterceptor(s.maxRetries),
serverinterceptors.UnaryCrashInterceptor, serverinterceptors.UnaryCrashInterceptor,
serverinterceptors.UnaryStatInterceptor(s.metrics), serverinterceptors.UnaryStatInterceptor(s.metrics),
serverinterceptors.UnaryPrometheusInterceptor, serverinterceptors.UnaryPrometheusInterceptor,
@@ -89,10 +87,3 @@ func WithMetrics(metrics *stat.Metrics) ServerOption {
options.metrics = metrics options.metrics = metrics
} }
} }
// WithMaxRetries returns a func that sets a max retries to a Server.
func WithMaxRetries(maxRetries int) ServerOption {
return func(options *rpcServerOptions) {
options.MaxRetries = maxRetries
}
}

6
zrpc/internal/server.go
View File

@@ -21,7 +21,6 @@ type (
baseRpcServer struct { baseRpcServer struct {
address string address string
metrics *stat.Metrics metrics *stat.Metrics
maxRetries int
options []grpc.ServerOption options []grpc.ServerOption
streamInterceptors []grpc.StreamServerInterceptor streamInterceptors []grpc.StreamServerInterceptor
unaryInterceptors []grpc.UnaryServerInterceptor unaryInterceptors []grpc.UnaryServerInterceptor
@@ -30,9 +29,8 @@ type (
func newBaseRpcServer(address string, rpcServerOpts *rpcServerOptions) *baseRpcServer { func newBaseRpcServer(address string, rpcServerOpts *rpcServerOptions) *baseRpcServer {
return &baseRpcServer{ return &baseRpcServer{
address: address, address: address,
metrics: rpcServerOpts.metrics, metrics: rpcServerOpts.metrics,
maxRetries: rpcServerOpts.MaxRetries,
} }
} }

35
zrpc/internal/serverinterceptors/retryinterceptor.go
View File

@@ -1,35 +0,0 @@
package serverinterceptors
import (
"context"
"strconv"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/core/retry"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
func RetryInterceptor(maxAttempt int) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo,
handler grpc.UnaryHandler) (resp interface{}, err error) {
var md metadata.MD
requestMd, ok := metadata.FromIncomingContext(ctx)
if ok {
md = requestMd.Copy()
attemptMd := md.Get(retry.AttemptMetadataKey)
if len(attemptMd) != 0 && attemptMd[0] != "" {
if attempt, err := strconv.Atoi(attemptMd[0]); err == nil {
if attempt > maxAttempt {
logx.WithContext(ctx).Errorf("retries exceeded:%d, max retries:%d", attempt, maxAttempt)
return nil, status.Error(codes.FailedPrecondition, "Retries exceeded")
}
}
}
}
return handler(ctx, req)
}
}

45
zrpc/internal/serverinterceptors/retryinterceptor_test.go
View File

@@ -1,45 +0,0 @@
package serverinterceptors
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/tal-tech/go-zero/core/retry"
"google.golang.org/grpc/metadata"
)
func TestRetryInterceptor(t *testing.T) {
t.Run("retries exceeded", func(t *testing.T) {
interceptor := RetryInterceptor(2)
ctx := metadata.NewIncomingContext(context.Background(),
metadata.New(map[string]string{retry.AttemptMetadataKey: "3"}))
resp, err := interceptor(ctx, nil, nil, func(ctx context.Context,
req interface{}) (interface{}, error) {
return nil, nil
})
assert.Error(t, err)
assert.Nil(t, resp)
})
t.Run("reasonable retries", func(t *testing.T) {
interceptor := RetryInterceptor(2)
ctx := metadata.NewIncomingContext(context.Background(),
metadata.New(map[string]string{retry.AttemptMetadataKey: "2"}))
resp, err := interceptor(ctx, nil, nil, func(ctx context.Context,
req interface{}) (interface{}, error) {
return nil, nil
})
assert.NoError(t, err)
assert.Nil(t, resp)
})
t.Run("no retries", func(t *testing.T) {
interceptor := RetryInterceptor(0)
resp, err := interceptor(context.Background(), nil, nil,
func(ctx context.Context, req interface{}) (interface{}, error) {
return nil, nil
})
assert.NoError(t, err)
assert.Nil(t, resp)
})
}

1
zrpc/server.go
View File

@@ -40,7 +40,6 @@ func NewServer(c RpcServerConf, register internal.RegisterFn) (*RpcServer, error
metrics := stat.NewMetrics(c.ListenOn) metrics := stat.NewMetrics(c.ListenOn)
serverOptions := []internal.ServerOption{ serverOptions := []internal.ServerOption{
internal.WithMetrics(metrics), internal.WithMetrics(metrics),
internal.WithMaxRetries(c.MaxRetries),
} }
if c.HasEtcd() { if c.HasEtcd() {