zeromicro/go-zero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add metrics (#3624)

Browse Source
This commit is contained in:
MarkJoyMa
2023-10-26 23:51:28 +08:00
committed by GitHub
parent 199e86050e
commit c05e03bb5a
14 changed files with 759 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
core/stores/redis/hook.go
View File

@@ -54,9 +54,10 @@ func (h hook) AfterProcess(ctx context.Context, cmd red.Cmder) error {
duration := timex.Since(start)
if duration > slowThreshold.Load() {
logDuration(ctx, []red.Cmder{cmd}, duration)
metricSlowCount.Inc(cmd.Name())
}
metricReqDur.Observe(duration.Milliseconds(), cmd.Name())
metricReqDur.ObserveFloat(float64(duration)/float64(time.Millisecond), cmd.Name())
if msg := formatError(err); len(msg) > 0 {
metricReqErr.Inc(cmd.Name(), msg)
}

170
core/stores/redis/metrics.go
View File

@@ -1,6 +1,12 @@
package redis
import "github.com/zeromicro/go-zero/core/metric"
import (
"sync"
red "github.com/go-redis/redis/v8"
"github.com/prometheus/client_golang/prometheus"
"github.com/zeromicro/go-zero/core/metric"
)
const namespace = "redis_client"
@@ -11,7 +17,7 @@ var (
Name: "duration_ms",
Help: "redis client requests duration(ms).",
Labels: []string{"command"},
Buckets: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500},
Buckets: []float64{0.25, 0.5, 1, 1.5, 2, 3, 5, 10, 25, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 15000},
})
metricReqErr = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: namespace,
@@ -20,4 +26,164 @@ var (
Help: "redis client requests error count.",
Labels: []string{"command", "error"},
})
metricSlowCount = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: namespace,
Subsystem: "requests",
Name: "slow_total",
Help: "redis client requests slow count.",
Labels: []string{"command"},
})
connLabels = []string{"key", "client_type"}
connCollector = newCollector()
_ prometheus.Collector = (*collector)(nil)
)
type (
statGetter struct {
clientType string
key string
poolSize int
poolStats func() *red.PoolStats
}
// collector collects statistics from a redis client.
// It implements the prometheus.Collector interface.
collector struct {
hitDesc *prometheus.Desc
missDesc *prometheus.Desc
timeoutDesc *prometheus.Desc
totalDesc *prometheus.Desc
idleDesc *prometheus.Desc
staleDesc *prometheus.Desc
maxDesc *prometheus.Desc
clients []*statGetter
lock sync.Mutex
}
)
func newCollector() *collector {
c := &collector{
hitDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "pool_hit_total"),
"Number of times a connection was found in the pool",
connLabels, nil,
),
missDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "pool_miss_total"),
"Number of times a connection was not found in the pool",
connLabels, nil,
),
timeoutDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "pool_timeout_total"),
"Number of times a timeout occurred when looking for a connection in the pool",
connLabels, nil,
),
totalDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "pool_conn_total_current"),
"Current number of connections in the pool",
connLabels, nil,
),
idleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "pool_conn_idle_current"),
"Current number of idle connections in the pool",
connLabels, nil,
),
staleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "pool_conn_stale_total"),
"Number of times a connection was removed from the pool because it was stale",
connLabels, nil,
),
maxDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "pool_conn_max"),
"Max number of connections in the pool",
connLabels, nil,
),
}
prometheus.MustRegister(c)
return c
}
// Describe implements the prometheus.Collector interface.
func (s *collector) Describe(descs chan<- *prometheus.Desc) {
descs <- s.hitDesc
descs <- s.missDesc
descs <- s.timeoutDesc
descs <- s.totalDesc
descs <- s.idleDesc
descs <- s.staleDesc
descs <- s.maxDesc
}
// Collect implements the prometheus.Collector interface.
func (s *collector) Collect(metrics chan<- prometheus.Metric) {
s.lock.Lock()
defer s.lock.Unlock()
for _, client := range s.clients {
key, clientType := client.key, client.clientType
stats := client.poolStats()
metrics <- prometheus.MustNewConstMetric(
s.hitDesc,
prometheus.CounterValue,
float64(stats.Hits),
key,
clientType,
)
metrics <- prometheus.MustNewConstMetric(
s.missDesc,
prometheus.CounterValue,
float64(stats.Misses),
key,
clientType,
)
metrics <- prometheus.MustNewConstMetric(
s.timeoutDesc,
prometheus.CounterValue,
float64(stats.Timeouts),
key,
clientType,
)
metrics <- prometheus.MustNewConstMetric(
s.totalDesc,
prometheus.GaugeValue,
float64(stats.TotalConns),
key,
clientType,
)
metrics <- prometheus.MustNewConstMetric(
s.idleDesc,
prometheus.GaugeValue,
float64(stats.IdleConns),
key,
clientType,
)
metrics <- prometheus.MustNewConstMetric(
s.staleDesc,
prometheus.CounterValue,
float64(stats.StaleConns),
key,
clientType,
)
metrics <- prometheus.MustNewConstMetric(
s.maxDesc,
prometheus.CounterValue,
float64(client.poolSize),
key,
clientType,
)
}
}
func (s *collector) registerClient(client *statGetter) {
s.lock.Lock()
defer s.lock.Unlock()
s.clients = append(s.clients, client)
}

130
core/stores/redis/metrics_test.go Normal file
View File

@@ -0,0 +1,130 @@
package redis
import (
"io"
"net/http"
"strings"
"testing"
"time"
red "github.com/go-redis/redis/v8"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/assert"
"github.com/zeromicro/go-zero/core/conf"
"github.com/zeromicro/go-zero/internal/devserver"
)
func TestRedisMetric(t *testing.T) {
cfg := devserver.Config{}
_ = conf.FillDefault(&cfg)
server := devserver.NewServer(cfg)
server.StartAsync()
time.Sleep(time.Second)
metricReqDur.Observe(8, "test-cmd")
metricReqErr.Inc("test-cmd", "internal-error")
metricSlowCount.Inc("test-cmd")
url := "http://127.0.0.1:6060/metrics"
resp, err := http.Get(url)
assert.Nil(t, err)
defer resp.Body.Close()
s, err := io.ReadAll(resp.Body)
assert.Nil(t, err)
content := string(s)
assert.Contains(t, content, "redis_client_requests_duration_ms_sum{command=\"test-cmd\"} 8\n")
assert.Contains(t, content, "redis_client_requests_duration_ms_count{command=\"test-cmd\"} 1\n")
assert.Contains(t, content, "redis_client_requests_error_total{command=\"test-cmd\",error=\"internal-error\"} 1\n")
assert.Contains(t, content, "redis_client_requests_slow_total{command=\"test-cmd\"} 1\n")
}
func Test_newCollector(t *testing.T) {
prometheus.Unregister(connCollector)
c := newCollector()
c.registerClient(&statGetter{
clientType: "node",
key: "test1",
poolSize: 10,
poolStats: func() *red.PoolStats {
return &red.PoolStats{
Hits: 10000,
Misses: 10,
Timeouts: 5,
TotalConns: 100,
IdleConns: 20,
StaleConns: 1,
}
},
})
c.registerClient(&statGetter{
clientType: "node",
key: "test2",
poolSize: 11,
poolStats: func() *red.PoolStats {
return &red.PoolStats{
Hits: 10001,
Misses: 11,
Timeouts: 6,
TotalConns: 101,
IdleConns: 21,
StaleConns: 2,
}
},
})
c.registerClient(&statGetter{
clientType: "cluster",
key: "test3",
poolSize: 5,
poolStats: func() *red.PoolStats {
return &red.PoolStats{
Hits: 20000,
Misses: 20,
Timeouts: 10,
TotalConns: 200,
IdleConns: 40,
StaleConns: 2,
}
},
})
val := `
# HELP redis_client_pool_conn_idle_current Current number of idle connections in the pool
# TYPE redis_client_pool_conn_idle_current gauge
redis_client_pool_conn_idle_current{client_type="cluster",key="test3"} 40
redis_client_pool_conn_idle_current{client_type="node",key="test1"} 20
redis_client_pool_conn_idle_current{client_type="node",key="test2"} 21
# HELP redis_client_pool_conn_max Max number of connections in the pool
# TYPE redis_client_pool_conn_max counter
redis_client_pool_conn_max{client_type="cluster",key="test3"} 5
redis_client_pool_conn_max{client_type="node",key="test1"} 10
redis_client_pool_conn_max{client_type="node",key="test2"} 11
# HELP redis_client_pool_conn_stale_total Number of times a connection was removed from the pool because it was stale
# TYPE redis_client_pool_conn_stale_total counter
redis_client_pool_conn_stale_total{client_type="cluster",key="test3"} 2
redis_client_pool_conn_stale_total{client_type="node",key="test1"} 1
redis_client_pool_conn_stale_total{client_type="node",key="test2"} 2
# HELP redis_client_pool_conn_total_current Current number of connections in the pool
# TYPE redis_client_pool_conn_total_current gauge
redis_client_pool_conn_total_current{client_type="cluster",key="test3"} 200
redis_client_pool_conn_total_current{client_type="node",key="test1"} 100
redis_client_pool_conn_total_current{client_type="node",key="test2"} 101
# HELP redis_client_pool_hit_total Number of times a connection was found in the pool
# TYPE redis_client_pool_hit_total counter
redis_client_pool_hit_total{client_type="cluster",key="test3"} 20000
redis_client_pool_hit_total{client_type="node",key="test1"} 10000
redis_client_pool_hit_total{client_type="node",key="test2"} 10001
# HELP redis_client_pool_miss_total Number of times a connection was not found in the pool
# TYPE redis_client_pool_miss_total counter
redis_client_pool_miss_total{client_type="cluster",key="test3"} 20
redis_client_pool_miss_total{client_type="node",key="test1"} 10
redis_client_pool_miss_total{client_type="node",key="test2"} 11
# HELP redis_client_pool_timeout_total Number of times a timeout occurred when looking for a connection in the pool
# TYPE redis_client_pool_timeout_total counter
redis_client_pool_timeout_total{client_type="cluster",key="test3"} 10
redis_client_pool_timeout_total{client_type="node",key="test1"} 5
redis_client_pool_timeout_total{client_type="node",key="test2"} 6
`
err := testutil.CollectAndCompare(c, strings.NewReader(val))
assert.NoError(t, err)
}

16
core/stores/redis/redisclientmanager.go
View File

@@ -3,6 +3,7 @@ package redis
import (
"crypto/tls"
"io"
"runtime"
red "github.com/go-redis/redis/v8"
"github.com/zeromicro/go-zero/core/syncx"
@@ -14,7 +15,11 @@ const (
idleConns = 8
)
var clientManager = syncx.NewResourceManager()
var (
clientManager = syncx.NewResourceManager()
// nodePoolSize is default pool size for node type of redis.
nodePoolSize = 10 * runtime.GOMAXPROCS(0)
)
func getClient(r *Redis) (*red.Client, error) {
val, err := clientManager.GetResource(r.Addr, func() (io.Closer, error) {
@@ -37,6 +42,15 @@ func getClient(r *Redis) (*red.Client, error) {
store.AddHook(hook)
}
connCollector.registerClient(&statGetter{
clientType: NodeType,
key: r.Addr,
poolSize: nodePoolSize,
poolStats: func() *red.PoolStats {
return store.PoolStats()
},
})
return store, nil
})
if err != nil {

16
core/stores/redis/redisclustermanager.go
View File

@@ -3,6 +3,7 @@ package redis
import (
"crypto/tls"
"io"
"runtime"
"strings"
red "github.com/go-redis/redis/v8"
@@ -11,7 +12,11 @@ import (
const addrSep = ","
var clusterManager = syncx.NewResourceManager()
var (
clusterManager = syncx.NewResourceManager()
// clusterPoolSize is default pool size for cluster type of redis.
clusterPoolSize = 5 * runtime.GOMAXPROCS(0)
)
func getCluster(r *Redis) (*red.ClusterClient, error) {
val, err := clusterManager.GetResource(r.Addr, func() (io.Closer, error) {
@@ -33,6 +38,15 @@ func getCluster(r *Redis) (*red.ClusterClient, error) {
store.AddHook(hook)
}
connCollector.registerClient(&statGetter{
clientType: ClusterType,
key: r.Addr,
poolSize: clusterPoolSize,
poolStats: func() *red.PoolStats {
return store.PoolStats()
},
})
return store, nil
})
if err != nil {