Add grpc retry (#1160)

* Add grpc retry

* Update grpc retry

* Add tests

* Fix a bug

* Add api && some tests

* Add comment

* Add double check

* Add server retry quota

* Update optimize code

* Fix bug

* Update optimize code

* Update optimize code

* Fix bug

zrpc/internal/rpcserver.go

@@ -14,7 +14,8 @@ type (
 	ServerOption func(options *rpcServerOptions)
 
 	rpcServerOptions struct {
-		metrics *stat.Metrics
+		metrics    *stat.Metrics
+		MaxRetries int
 	}
 
 	rpcServer struct {
@@ -38,7 +39,7 @@ func NewRpcServer(address string, opts ...ServerOption) Server {
 	}
 
 	return &rpcServer{
-		baseRpcServer: newBaseRpcServer(address, options.metrics),
+		baseRpcServer: newBaseRpcServer(address, &options),
 	}
 }
 
@@ -55,6 +56,7 @@ func (s *rpcServer) Start(register RegisterFn) error {
 
 	unaryInterceptors := []grpc.UnaryServerInterceptor{
 		serverinterceptors.UnaryTracingInterceptor,
+		serverinterceptors.RetryInterceptor(s.maxRetries),
 		serverinterceptors.UnaryCrashInterceptor,
 		serverinterceptors.UnaryStatInterceptor(s.metrics),
 		serverinterceptors.UnaryPrometheusInterceptor,
@@ -87,3 +89,10 @@ func WithMetrics(metrics *stat.Metrics) ServerOption {
 		options.metrics = metrics
 	}
 }
+
+// WithMaxRetries returns a func that sets a max retries to a Server.
+func WithMaxRetries(maxRetries int) ServerOption {
+	return func(options *rpcServerOptions) {
+		options.MaxRetries = maxRetries
+	}
+}