feat: mysql and redis metric support (#2355)

* feat: mysql and redis metric support

* feat: mysql and redis metric support

* feat: mysql and redis metric support

Co-authored-by: dawn.zhou <dawn.zhou@yijinin.com>

core/stores/redis/hook.go

@@ -56,6 +56,11 @@ func (h hook) AfterProcess(ctx context.Context, cmd red.Cmder) error {
 		logDuration(ctx, cmd, duration)
 	}
 
+	metricReqDur.Observe(int64(duration/time.Millisecond), cmd.Name())
+	if msg := errFormat(err); len(msg) > 0 {
+		metricReqErr.Inc(cmd.Name(), msg)
+	}
+
 	return nil
 }
 
@@ -98,9 +103,43 @@ func (h hook) AfterProcessPipeline(ctx context.Context, cmds []red.Cmder) error
 		logDuration(ctx, cmds[0], duration)
 	}
 
+	metricReqDur.Observe(int64(duration/time.Millisecond), "Pipeline")
+	if msg := errFormat(batchError.Err()); len(msg) > 0 {
+		metricReqErr.Inc("Pipeline", msg)
+	}
+
 	return nil
 }
 
+func errFormat(err error) string {
+	if err == nil || err == red.Nil {
+		return ""
+	}
+
+	es := err.Error()
+	switch {
+	case strings.HasPrefix(es, "read"):
+		return "read timeout"
+	case strings.HasPrefix(es, "dial"):
+		if strings.Contains(es, "connection refused") {
+			return "connection refused"
+		}
+		return "dial timeout"
+	case strings.HasPrefix(es, "write"):
+		return "write timeout"
+	case strings.Contains(es, "EOF"):
+		return "eof"
+	case strings.Contains(es, "reset"):
+		return "reset"
+	case strings.Contains(es, "broken"):
+		return "broken pipe"
+	case strings.Contains(es, "breaker"):
+		return "breaker"
+	default:
+		return "unexpected error"
+	}
+}
+
 func logDuration(ctx context.Context, cmd red.Cmder, duration time.Duration) {
 	var buf strings.Builder
 	for i, arg := range cmd.Args() {

core/stores/redis/metrics.go

@@ -0,0 +1,23 @@
+package redis
+
+import "github.com/zeromicro/go-zero/core/metric"
+
+const namespace = "redis_client"
+
+var (
+	metricReqDur = metric.NewHistogramVec(&metric.HistogramVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "duration_ms",
+		Help:      "redis client requests duration(ms).",
+		Labels:    []string{"command"},
+		Buckets:   []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500},
+	})
+	metricReqErr = metric.NewCounterVec(&metric.CounterVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "error_total",
+		Help:      "redis client requests error count.",
+		Labels:    []string{"command", "error"},
+	})
+)

core/stores/sqlx/metrics.go

@@ -0,0 +1,23 @@
+package sqlx
+
+import "github.com/zeromicro/go-zero/core/metric"
+
+const namespace = "mysql_client"
+
+var (
+	metricReqDur = metric.NewHistogramVec(&metric.HistogramVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "durations_ms",
+		Help:      "mysql client requests duration(ms).",
+		Labels:    []string{"command"},
+		Buckets:   []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500},
+	})
+	metricReqErr = metric.NewCounterVec(&metric.CounterVecOpts{
+		Namespace: namespace,
+		Subsystem: "requests",
+		Name:      "error_total",
+		Help:      "mysql client requests error count.",
+		Labels:    []string{"command", "error"},
+	})
+)

core/stores/sqlx/sqlconn.go

@@ -154,6 +154,10 @@ func (db *commonSqlConn) ExecCtx(ctx context.Context, q string, args ...interfac
 		return err
 	}, db.acceptable)
 
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("Exec", "breaker")
+	}
+
 	return
 }
 
@@ -187,6 +191,10 @@ func (db *commonSqlConn) PrepareCtx(ctx context.Context, query string) (stmt Stm
 		return nil
 	}, db.acceptable)
 
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("Prepare", "breaker")
+	}
+
 	return
 }
 
@@ -270,9 +278,15 @@ func (db *commonSqlConn) TransactCtx(ctx context.Context, fn func(context.Contex
 		endSpan(span, err)
 	}()
 
-	return db.brk.DoWithAcceptable(func() error {
+	err = db.brk.DoWithAcceptable(func() error {
 		return transact(ctx, db, db.beginTx, fn)
 	}, db.acceptable)
+
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("Transact", "breaker")
+	}
+
+	return
 }
 
 func (db *commonSqlConn) acceptable(err error) bool {
@@ -287,7 +301,7 @@ func (db *commonSqlConn) acceptable(err error) bool {
 func (db *commonSqlConn) queryRows(ctx context.Context, scanner func(*sql.Rows) error,
 	q string, args ...interface{}) (err error) {
 	var qerr error
-	return db.brk.DoWithAcceptable(func() error {
+	err = db.brk.DoWithAcceptable(func() error {
 		conn, err := db.connProv()
 		if err != nil {
 			db.onError(err)
@@ -301,6 +315,12 @@ func (db *commonSqlConn) queryRows(ctx context.Context, scanner func(*sql.Rows)
 	}, func(err error) bool {
 		return qerr == err || db.acceptable(err)
 	})
+
+	if err == breaker.ErrServiceUnavailable {
+		metricReqErr.Inc("queryRows", "breaker")
+	}
+
+	return
 }
 
 func (s statement) Close() error {

core/stores/sqlx/sqlconn_test.go

@@ -17,7 +17,8 @@ func init() {
 }
 
 func TestSqlConn(t *testing.T) {
-	mock := buildConn()
+	mock, err := buildConn()
+	assert.Nil(t, err)
 	mock.ExpectExec("any")
 	mock.ExpectQuery("any").WillReturnRows(sqlmock.NewRows([]string{"foo"}))
 	conn := NewMysql(mockedDatasource)
@@ -50,8 +51,8 @@ func TestSqlConn(t *testing.T) {
 	}))
 }
 
-func buildConn() (mock sqlmock.Sqlmock) {
-	connManager.GetResource(mockedDatasource, func() (io.Closer, error) {
+func buildConn() (mock sqlmock.Sqlmock, err error) {
+	_, err = connManager.GetResource(mockedDatasource, func() (io.Closer, error) {
 		var db *sql.DB
 		var err error
 		db, mock, err = sqlmock.New()

core/stores/sqlx/stmt.go

@@ -135,6 +135,8 @@ func (e *realSqlGuard) finish(ctx context.Context, err error) {
 	if err != nil {
 		logSqlError(ctx, e.stmt, err)
 	}
+
+	metricReqDur.Observe(int64(duration/time.Millisecond), e.command)
 }
 
 func (e *realSqlGuard) start(q string, args ...interface{}) error {