feat: add metrics (#3624)
This commit is contained in:
MarkJoyMa
GitHub
@@ -1,8 +1,14 @@
|
||||
package sqlx
|
||||
|
||||
import "github.com/zeromicro/go-zero/core/metric"
|
||||
import (
|
||||
"database/sql"
|
||||
"sync"
|
||||
|
||||
const namespace = "sql_client"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/zeromicro/go-zero/core/metric"
|
||||
)
|
||||
|
||||
const namespace = "mysql_client"
|
||||
|
||||
var (
|
||||
metricReqDur = metric.NewHistogramVec(&metric.HistogramVecOpts{
|
||||
@@ -11,7 +17,7 @@ var (
|
||||
Name: "duration_ms",
|
||||
Help: "mysql client requests duration(ms).",
|
||||
Labels: []string{"command"},
|
||||
Buckets: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500},
|
||||
Buckets: []float64{0.25, 0.5, 1, 1.5, 2, 3, 5, 10, 25, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 15000},
|
||||
})
|
||||
metricReqErr = metric.NewCounterVec(&metric.CounterVecOpts{
|
||||
Namespace: namespace,
|
||||
@@ -20,4 +26,138 @@ var (
|
||||
Help: "mysql client requests error count.",
|
||||
Labels: []string{"command", "error"},
|
||||
})
|
||||
metricSlowCount = metric.NewCounterVec(&metric.CounterVecOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: "requests",
|
||||
Name: "slow_total",
|
||||
Help: "mysql client requests slow count.",
|
||||
Labels: []string{"command"},
|
||||
})
|
||||
|
||||
connLabels = []string{"db_name", "hash"}
|
||||
|
||||
connCollector = newCollector()
|
||||
|
||||
_ prometheus.Collector = (*collector)(nil)
|
||||
)
|
||||
|
||||
type (
|
||||
statGetter struct {
|
||||
dbName string
|
||||
hash string
|
||||
poolStats func() sql.DBStats
|
||||
}
|
||||
|
||||
// collector collects statistics from a redis client.
|
||||
// It implements the prometheus.Collector interface.
|
||||
collector struct {
|
||||
maxOpenConnections *prometheus.Desc
|
||||
|
||||
openConnections *prometheus.Desc
|
||||
inUseConnections *prometheus.Desc
|
||||
idleConnections *prometheus.Desc
|
||||
|
||||
waitCount *prometheus.Desc
|
||||
waitDuration *prometheus.Desc
|
||||
maxIdleClosed *prometheus.Desc
|
||||
maxIdleTimeClosed *prometheus.Desc
|
||||
maxLifetimeClosed *prometheus.Desc
|
||||
|
||||
clients []*statGetter
|
||||
lock sync.Mutex
|
||||
}
|
||||
)
|
||||
|
||||
func newCollector() *collector {
|
||||
c := &collector{
|
||||
maxOpenConnections: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "max_open_connections"),
|
||||
"Maximum number of open connections to the database.",
|
||||
connLabels, nil,
|
||||
),
|
||||
openConnections: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "open_connections"),
|
||||
"The number of established connections both in use and idle.",
|
||||
connLabels, nil,
|
||||
),
|
||||
inUseConnections: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "in_use_connections"),
|
||||
"The number of connections currently in use.",
|
||||
connLabels, nil,
|
||||
),
|
||||
idleConnections: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "idle_connections"),
|
||||
"The number of idle connections.",
|
||||
connLabels, nil,
|
||||
),
|
||||
waitCount: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "wait_count_total"),
|
||||
"The total number of connections waited for.",
|
||||
connLabels, nil,
|
||||
),
|
||||
waitDuration: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "wait_duration_seconds_total"),
|
||||
"The total time blocked waiting for a new connection.",
|
||||
connLabels, nil,
|
||||
),
|
||||
maxIdleClosed: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "max_idle_closed_total"),
|
||||
"The total number of connections closed due to SetMaxIdleConns.",
|
||||
connLabels, nil,
|
||||
),
|
||||
maxIdleTimeClosed: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "max_idle_time_closed_total"),
|
||||
"The total number of connections closed due to SetConnMaxIdleTime.",
|
||||
connLabels, nil,
|
||||
),
|
||||
maxLifetimeClosed: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "max_lifetime_closed_total"),
|
||||
"The total number of connections closed due to SetConnMaxLifetime.",
|
||||
connLabels, nil,
|
||||
),
|
||||
}
|
||||
|
||||
prometheus.MustRegister(c)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// Describe implements the prometheus.Collector interface.
|
||||
func (c *collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
ch <- c.maxOpenConnections
|
||||
ch <- c.openConnections
|
||||
ch <- c.inUseConnections
|
||||
ch <- c.idleConnections
|
||||
ch <- c.waitCount
|
||||
ch <- c.waitDuration
|
||||
ch <- c.maxIdleClosed
|
||||
ch <- c.maxLifetimeClosed
|
||||
ch <- c.maxIdleTimeClosed
|
||||
}
|
||||
|
||||
// Collect implements the prometheus.Collector interface.
|
||||
func (c *collector) Collect(ch chan<- prometheus.Metric) {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
for _, client := range c.clients {
|
||||
dbName, hash := client.dbName, client.hash
|
||||
stats := client.poolStats()
|
||||
ch <- prometheus.MustNewConstMetric(c.maxOpenConnections, prometheus.GaugeValue, float64(stats.MaxOpenConnections), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.openConnections, prometheus.GaugeValue, float64(stats.OpenConnections), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.inUseConnections, prometheus.GaugeValue, float64(stats.InUse), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.idleConnections, prometheus.GaugeValue, float64(stats.Idle), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.waitCount, prometheus.CounterValue, float64(stats.WaitCount), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.waitDuration, prometheus.CounterValue, stats.WaitDuration.Seconds(), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.maxIdleClosed, prometheus.CounterValue, float64(stats.MaxIdleClosed), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.maxLifetimeClosed, prometheus.CounterValue, float64(stats.MaxLifetimeClosed), dbName, hash)
|
||||
ch <- prometheus.MustNewConstMetric(c.maxIdleTimeClosed, prometheus.CounterValue, float64(stats.MaxIdleTimeClosed), dbName, hash)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *collector) registerClient(client *statGetter) {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
c.clients = append(c.clients, client)
|
||||
}
|
||||
|
||||
147
core/stores/sqlx/metrics_test.go
Normal file
147
core/stores/sqlx/metrics_test.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package sqlx
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zeromicro/go-zero/core/conf"
|
||||
"github.com/zeromicro/go-zero/internal/devserver"
|
||||
)
|
||||
|
||||
func TestSqlxMetric(t *testing.T) {
|
||||
cfg := devserver.Config{}
|
||||
_ = conf.FillDefault(&cfg)
|
||||
cfg.Port = 6480
|
||||
server := devserver.NewServer(cfg)
|
||||
server.StartAsync()
|
||||
time.Sleep(time.Second)
|
||||
|
||||
metricReqDur.Observe(8, "test-cmd")
|
||||
metricReqErr.Inc("test-cmd", "internal-error")
|
||||
metricSlowCount.Inc("test-cmd")
|
||||
|
||||
url := "http://127.0.0.1:6480/metrics"
|
||||
resp, err := http.Get(url)
|
||||
assert.Nil(t, err)
|
||||
defer resp.Body.Close()
|
||||
s, err := io.ReadAll(resp.Body)
|
||||
assert.Nil(t, err)
|
||||
content := string(s)
|
||||
assert.Contains(t, content, "mysql_client_requests_duration_ms_sum{command=\"test-cmd\"} 8\n")
|
||||
assert.Contains(t, content, "mysql_client_requests_duration_ms_count{command=\"test-cmd\"} 1\n")
|
||||
assert.Contains(t, content, "mysql_client_requests_error_total{command=\"test-cmd\",error=\"internal-error\"} 1\n")
|
||||
assert.Contains(t, content, "mysql_client_requests_slow_total{command=\"test-cmd\"} 1\n")
|
||||
}
|
||||
|
||||
func TestMetricCollector(t *testing.T) {
|
||||
prometheus.Unregister(connCollector)
|
||||
c := newCollector()
|
||||
c.registerClient(&statGetter{
|
||||
dbName: "db-1",
|
||||
hash: "hash-1",
|
||||
poolStats: func() sql.DBStats {
|
||||
return sql.DBStats{
|
||||
MaxOpenConnections: 1,
|
||||
OpenConnections: 2,
|
||||
InUse: 3,
|
||||
Idle: 4,
|
||||
WaitCount: 5,
|
||||
WaitDuration: 6 * time.Second,
|
||||
MaxIdleClosed: 7,
|
||||
MaxIdleTimeClosed: 8,
|
||||
MaxLifetimeClosed: 9,
|
||||
}
|
||||
},
|
||||
})
|
||||
c.registerClient(&statGetter{
|
||||
dbName: "db-1",
|
||||
hash: "hash-2",
|
||||
poolStats: func() sql.DBStats {
|
||||
return sql.DBStats{
|
||||
MaxOpenConnections: 10,
|
||||
OpenConnections: 20,
|
||||
InUse: 30,
|
||||
Idle: 40,
|
||||
WaitCount: 50,
|
||||
WaitDuration: 60 * time.Second,
|
||||
MaxIdleClosed: 70,
|
||||
MaxIdleTimeClosed: 80,
|
||||
MaxLifetimeClosed: 90,
|
||||
}
|
||||
},
|
||||
})
|
||||
c.registerClient(&statGetter{
|
||||
dbName: "db-2",
|
||||
hash: "hash-2",
|
||||
poolStats: func() sql.DBStats {
|
||||
return sql.DBStats{
|
||||
MaxOpenConnections: 100,
|
||||
OpenConnections: 200,
|
||||
InUse: 300,
|
||||
Idle: 400,
|
||||
WaitCount: 500,
|
||||
WaitDuration: 600 * time.Second,
|
||||
MaxIdleClosed: 700,
|
||||
MaxIdleTimeClosed: 800,
|
||||
MaxLifetimeClosed: 900,
|
||||
}
|
||||
},
|
||||
})
|
||||
val := `
|
||||
# HELP mysql_client_idle_connections The number of idle connections.
|
||||
# TYPE mysql_client_idle_connections gauge
|
||||
mysql_client_idle_connections{db_name="db-1",hash="hash-1"} 4
|
||||
mysql_client_idle_connections{db_name="db-1",hash="hash-2"} 40
|
||||
mysql_client_idle_connections{db_name="db-2",hash="hash-2"} 400
|
||||
# HELP mysql_client_in_use_connections The number of connections currently in use.
|
||||
# TYPE mysql_client_in_use_connections gauge
|
||||
mysql_client_in_use_connections{db_name="db-1",hash="hash-1"} 3
|
||||
mysql_client_in_use_connections{db_name="db-1",hash="hash-2"} 30
|
||||
mysql_client_in_use_connections{db_name="db-2",hash="hash-2"} 300
|
||||
# HELP mysql_client_max_idle_closed_total The total number of connections closed due to SetMaxIdleConns.
|
||||
# TYPE mysql_client_max_idle_closed_total counter
|
||||
mysql_client_max_idle_closed_total{db_name="db-1",hash="hash-1"} 7
|
||||
mysql_client_max_idle_closed_total{db_name="db-1",hash="hash-2"} 70
|
||||
mysql_client_max_idle_closed_total{db_name="db-2",hash="hash-2"} 700
|
||||
# HELP mysql_client_max_idle_time_closed_total The total number of connections closed due to SetConnMaxIdleTime.
|
||||
# TYPE mysql_client_max_idle_time_closed_total counter
|
||||
mysql_client_max_idle_time_closed_total{db_name="db-1",hash="hash-1"} 8
|
||||
mysql_client_max_idle_time_closed_total{db_name="db-1",hash="hash-2"} 80
|
||||
mysql_client_max_idle_time_closed_total{db_name="db-2",hash="hash-2"} 800
|
||||
# HELP mysql_client_max_lifetime_closed_total The total number of connections closed due to SetConnMaxLifetime.
|
||||
# TYPE mysql_client_max_lifetime_closed_total counter
|
||||
mysql_client_max_lifetime_closed_total{db_name="db-1",hash="hash-1"} 9
|
||||
mysql_client_max_lifetime_closed_total{db_name="db-1",hash="hash-2"} 90
|
||||
mysql_client_max_lifetime_closed_total{db_name="db-2",hash="hash-2"} 900
|
||||
# HELP mysql_client_max_open_connections Maximum number of open connections to the database.
|
||||
# TYPE mysql_client_max_open_connections gauge
|
||||
mysql_client_max_open_connections{db_name="db-1",hash="hash-1"} 1
|
||||
mysql_client_max_open_connections{db_name="db-1",hash="hash-2"} 10
|
||||
mysql_client_max_open_connections{db_name="db-2",hash="hash-2"} 100
|
||||
# HELP mysql_client_open_connections The number of established connections both in use and idle.
|
||||
# TYPE mysql_client_open_connections gauge
|
||||
mysql_client_open_connections{db_name="db-1",hash="hash-1"} 2
|
||||
mysql_client_open_connections{db_name="db-1",hash="hash-2"} 20
|
||||
mysql_client_open_connections{db_name="db-2",hash="hash-2"} 200
|
||||
# HELP mysql_client_wait_count_total The total number of connections waited for.
|
||||
# TYPE mysql_client_wait_count_total counter
|
||||
mysql_client_wait_count_total{db_name="db-1",hash="hash-1"} 5
|
||||
mysql_client_wait_count_total{db_name="db-1",hash="hash-2"} 50
|
||||
mysql_client_wait_count_total{db_name="db-2",hash="hash-2"} 500
|
||||
# HELP mysql_client_wait_duration_seconds_total The total time blocked waiting for a new connection.
|
||||
# TYPE mysql_client_wait_duration_seconds_total counter
|
||||
mysql_client_wait_duration_seconds_total{db_name="db-1",hash="hash-1"} 6
|
||||
mysql_client_wait_duration_seconds_total{db_name="db-1",hash="hash-2"} 60
|
||||
mysql_client_wait_duration_seconds_total{db_name="db-2",hash="hash-2"} 600
|
||||
`
|
||||
|
||||
err := testutil.CollectAndCompare(c, strings.NewReader(val))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
@@ -128,6 +128,7 @@ func (e *realSqlGuard) finish(ctx context.Context, err error) {
|
||||
duration := timex.Since(e.startTime)
|
||||
if duration > slowThreshold.Load() {
|
||||
logx.WithContext(ctx).WithDuration(duration).Slowf("[SQL] %s: slowcall - %s", e.command, e.stmt)
|
||||
metricSlowCount.Inc(e.command)
|
||||
} else if logSql.True() {
|
||||
logx.WithContext(ctx).WithDuration(duration).Infof("sql %s: %s", e.command, e.stmt)
|
||||
}
|
||||
@@ -136,7 +137,7 @@ func (e *realSqlGuard) finish(ctx context.Context, err error) {
|
||||
logSqlError(ctx, e.stmt, err)
|
||||
}
|
||||
|
||||
metricReqDur.Observe(duration.Milliseconds(), e.command)
|
||||
metricReqDur.ObserveFloat(float64(duration)/float64(time.Millisecond), e.command)
|
||||
}
|
||||
|
||||
func (e *realSqlGuard) start(q string, args ...any) error {
|
||||
|
||||
Reference in New Issue
Block a user