Compare commits
95 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
456b395860 | ||
|
|
f3c367a323 | ||
|
|
a32028c4fb | ||
|
|
b4572fa064 | ||
|
|
ccbabf6f58 | ||
|
|
5989444227 | ||
|
|
dc286a03f5 | ||
|
|
b82c02ed16 | ||
|
|
59ba4ecc5b | ||
|
|
5e7b514ae2 | ||
|
|
2b1466e41e | ||
|
|
9c9f80518f | ||
|
|
25973d6b59 | ||
|
|
6237d01948 | ||
|
|
49316b113e | ||
|
|
6a673e8cb0 | ||
|
|
0efa28ddbd | ||
|
|
0b6a13fe84 | ||
|
|
11aa6668e8 | ||
|
|
267a283328 | ||
|
|
2d8366b30e | ||
|
|
db83843558 | ||
|
|
50565c9765 | ||
|
|
4c02a19a14 | ||
|
|
a1b990c5ec | ||
|
|
2607bb8863 | ||
|
|
5bf37535fe | ||
|
|
ed85775fd5 | ||
|
|
418f8f6666 | ||
|
|
22e75cdf78 | ||
|
|
e79c42add1 | ||
|
|
9e14820698 | ||
|
|
2ebb5b6b58 | ||
|
|
2673dbc6e1 | ||
|
|
d21d770b5b | ||
|
|
1252bd9cde | ||
|
|
054d9b5540 | ||
|
|
f03cfb0ff7 | ||
|
|
0214161bfc | ||
|
|
d4e38cb7f0 | ||
|
|
693a8b627a | ||
|
|
701208b6f4 | ||
|
|
b65fcc5512 | ||
|
|
3321ed3519 | ||
|
|
5e007c1f9f | ||
|
|
de2f8c06fb | ||
|
|
926d746df5 | ||
|
|
4b636cd293 | ||
|
|
4bdf5e4c90 | ||
|
|
721b7def7c | ||
|
|
f294090130 | ||
|
|
489980ea0f | ||
|
|
e12c8ae993 | ||
|
|
21aad62513 | ||
|
|
0b08aca554 | ||
|
|
6ef1b5e14c | ||
|
|
8745039877 | ||
|
|
9d9399ad10 | ||
|
|
e7dd04701c | ||
|
|
a3d7474ae0 | ||
|
|
6fdee77fa9 | ||
|
|
5f084fb7d2 | ||
|
|
c8ff9d2f23 | ||
|
|
78f5e7df87 | ||
|
|
6d8dc4630f | ||
|
|
03ac41438f | ||
|
|
4ef0b0a8ac | ||
|
|
87b1fba46c | ||
|
|
cfa6644b0c | ||
|
|
fcaebd73fb | ||
|
|
a26fc2b672 | ||
|
|
05c8dd0b9c | ||
|
|
d6c7da521e | ||
|
|
96b6d2ab58 | ||
|
|
88a73f1042 | ||
|
|
9428fface2 | ||
|
|
c637f86817 | ||
|
|
d4097af627 | ||
|
|
7da31921c7 | ||
|
|
47440964cd | ||
|
|
80d55dbc02 | ||
|
|
b541403ce2 | ||
|
|
a7c02414f3 | ||
|
|
196475383b | ||
|
|
fd75f700a2 | ||
|
|
d408a0d49b | ||
|
|
69d113a46d | ||
|
|
63c7f44a5f | ||
|
|
19888b7d11 | ||
|
|
d117e31993 | ||
|
|
10cd6053bc | ||
|
|
d1529fced8 | ||
|
|
4f59fd306a | ||
|
|
f77c73eec1 | ||
|
|
9b0b958f43 |
4
.codecov.yml
Normal file
4
.codecov.yml
Normal file
@@ -0,0 +1,4 @@
|
||||
ignore:
|
||||
- "doc"
|
||||
- "example"
|
||||
- "tools"
|
||||
11
.github/workflows/go.yml
vendored
11
.github/workflows/go.yml
vendored
@@ -25,10 +25,11 @@ jobs:
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go get -v -t -d ./...
|
||||
if [ -f Gopkg.toml ]; then
|
||||
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
|
||||
dep ensure
|
||||
fi
|
||||
|
||||
- name: Test
|
||||
run: go test -v -race ./...
|
||||
run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
|
||||
|
||||
- name: Codecov
|
||||
uses: codecov/codecov-action@v1.0.6
|
||||
with:
|
||||
token: ${{secrets.CODECOV_TOKEN}}
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
stages:
|
||||
- analysis
|
||||
|
||||
variables:
|
||||
GOPATH: '/runner-cache/zero'
|
||||
GOCACHE: '/runner-cache/zero'
|
||||
GOPROXY: 'https://goproxy.cn,direct'
|
||||
|
||||
analysis:
|
||||
stage: analysis
|
||||
image: golang
|
||||
script:
|
||||
- go version && go env
|
||||
- go test -short $(go list ./...) | grep -v "no test"
|
||||
only:
|
||||
- merge_requests
|
||||
tags:
|
||||
- common
|
||||
@@ -60,17 +60,15 @@ func do(name string, execute func(b Breaker) error) error {
|
||||
lock.RUnlock()
|
||||
if ok {
|
||||
return execute(b)
|
||||
} else {
|
||||
lock.Lock()
|
||||
b, ok = breakers[name]
|
||||
if ok {
|
||||
lock.Unlock()
|
||||
return execute(b)
|
||||
} else {
|
||||
b = NewBreaker(WithName(name))
|
||||
breakers[name] = b
|
||||
lock.Unlock()
|
||||
return execute(b)
|
||||
}
|
||||
}
|
||||
|
||||
lock.Lock()
|
||||
b, ok = breakers[name]
|
||||
if !ok {
|
||||
b = NewBreaker(WithName(name))
|
||||
breakers[name] = b
|
||||
}
|
||||
lock.Unlock()
|
||||
|
||||
return execute(b)
|
||||
}
|
||||
|
||||
@@ -72,9 +72,9 @@ func TestCacheWithLruEvicts(t *testing.T) {
|
||||
cache.Set("third", "third element")
|
||||
cache.Set("fourth", "fourth element")
|
||||
|
||||
value, ok := cache.Get("first")
|
||||
_, ok := cache.Get("first")
|
||||
assert.False(t, ok)
|
||||
value, ok = cache.Get("second")
|
||||
value, ok := cache.Get("second")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "second element", value)
|
||||
value, ok = cache.Get("third")
|
||||
@@ -94,9 +94,9 @@ func TestCacheWithLruEvicted(t *testing.T) {
|
||||
cache.Set("third", "third element")
|
||||
cache.Set("fourth", "fourth element")
|
||||
|
||||
value, ok := cache.Get("first")
|
||||
_, ok := cache.Get("first")
|
||||
assert.False(t, ok)
|
||||
value, ok = cache.Get("second")
|
||||
value, ok := cache.Get("second")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "second element", value)
|
||||
cache.Set("fifth", "fifth element")
|
||||
|
||||
@@ -213,7 +213,10 @@ func TestTimingWheel_SetTimer(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(stringx.RandId(), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var count int32
|
||||
ticker := timex.NewFakeTicker()
|
||||
tick := func() {
|
||||
@@ -291,7 +294,10 @@ func TestTimingWheel_SetAndMoveThenStart(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(stringx.RandId(), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var count int32
|
||||
ticker := timex.NewFakeTicker()
|
||||
tick := func() {
|
||||
@@ -376,7 +382,10 @@ func TestTimingWheel_SetAndMoveTwice(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(stringx.RandId(), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var count int32
|
||||
ticker := timex.NewFakeTicker()
|
||||
tick := func() {
|
||||
@@ -454,7 +463,10 @@ func TestTimingWheel_ElapsedAndSet(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(stringx.RandId(), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var count int32
|
||||
ticker := timex.NewFakeTicker()
|
||||
tick := func() {
|
||||
@@ -542,7 +554,10 @@ func TestTimingWheel_ElapsedAndSetThenMove(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(stringx.RandId(), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var count int32
|
||||
ticker := timex.NewFakeTicker()
|
||||
tick := func() {
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestContextCancel(t *testing.T) {
|
||||
assert.NotEqual(t, context.Canceled, c2.Err())
|
||||
}
|
||||
|
||||
func TestConextDeadline(t *testing.T) {
|
||||
func TestContextDeadline(t *testing.T) {
|
||||
c, _ := context.WithDeadline(context.Background(), time.Now().Add(10*time.Millisecond))
|
||||
o := ValueOnlyFrom(c)
|
||||
select {
|
||||
|
||||
@@ -2,7 +2,7 @@ package discov
|
||||
|
||||
import (
|
||||
"github.com/tal-tech/go-zero/core/discov/internal"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
)
|
||||
|
||||
type (
|
||||
@@ -26,7 +26,7 @@ func NewFacade(endpoints []string) Facade {
|
||||
|
||||
func (f Facade) Client() internal.EtcdClient {
|
||||
conn, err := f.registry.GetConn(f.endpoints)
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
return conn
|
||||
}
|
||||
|
||||
|
||||
4
core/discov/kubernetes/discov-namespace.yaml
Normal file
4
core/discov/kubernetes/discov-namespace.yaml
Normal file
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: discov
|
||||
368
core/discov/kubernetes/etcd.yaml
Normal file
368
core/discov/kubernetes/etcd.yaml
Normal file
@@ -0,0 +1,368 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: etcd
|
||||
namespace: discov
|
||||
spec:
|
||||
ports:
|
||||
- name: etcd-port
|
||||
port: 2379
|
||||
protocol: TCP
|
||||
targetPort: 2379
|
||||
selector:
|
||||
app: etcd
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
app: etcd
|
||||
etcd_node: etcd0
|
||||
name: etcd0
|
||||
namespace: discov
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /usr/local/bin/etcd
|
||||
- --name
|
||||
- etcd0
|
||||
- --initial-advertise-peer-urls
|
||||
- http://etcd0:2380
|
||||
- --listen-peer-urls
|
||||
- http://0.0.0.0:2380
|
||||
- --listen-client-urls
|
||||
- http://0.0.0.0:2379
|
||||
- --advertise-client-urls
|
||||
- http://etcd0:2379
|
||||
- --initial-cluster
|
||||
- etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
|
||||
- --initial-cluster-state
|
||||
- new
|
||||
image: quay.io/coreos/etcd:latest
|
||||
name: etcd0
|
||||
ports:
|
||||
- containerPort: 2379
|
||||
name: client
|
||||
protocol: TCP
|
||||
- containerPort: 2380
|
||||
name: server
|
||||
protocol: TCP
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- etcd
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
restartPolicy: Always
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
etcd_node: etcd0
|
||||
name: etcd0
|
||||
namespace: discov
|
||||
spec:
|
||||
ports:
|
||||
- name: client
|
||||
port: 2379
|
||||
protocol: TCP
|
||||
targetPort: 2379
|
||||
- name: server
|
||||
port: 2380
|
||||
protocol: TCP
|
||||
targetPort: 2380
|
||||
selector:
|
||||
etcd_node: etcd0
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
app: etcd
|
||||
etcd_node: etcd1
|
||||
name: etcd1
|
||||
namespace: discov
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /usr/local/bin/etcd
|
||||
- --name
|
||||
- etcd1
|
||||
- --initial-advertise-peer-urls
|
||||
- http://etcd1:2380
|
||||
- --listen-peer-urls
|
||||
- http://0.0.0.0:2380
|
||||
- --listen-client-urls
|
||||
- http://0.0.0.0:2379
|
||||
- --advertise-client-urls
|
||||
- http://etcd1:2379
|
||||
- --initial-cluster
|
||||
- etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
|
||||
- --initial-cluster-state
|
||||
- new
|
||||
image: quay.io/coreos/etcd:latest
|
||||
name: etcd1
|
||||
ports:
|
||||
- containerPort: 2379
|
||||
name: client
|
||||
protocol: TCP
|
||||
- containerPort: 2380
|
||||
name: server
|
||||
protocol: TCP
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- etcd
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
restartPolicy: Always
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
etcd_node: etcd1
|
||||
name: etcd1
|
||||
namespace: discov
|
||||
spec:
|
||||
ports:
|
||||
- name: client
|
||||
port: 2379
|
||||
protocol: TCP
|
||||
targetPort: 2379
|
||||
- name: server
|
||||
port: 2380
|
||||
protocol: TCP
|
||||
targetPort: 2380
|
||||
selector:
|
||||
etcd_node: etcd1
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
app: etcd
|
||||
etcd_node: etcd2
|
||||
name: etcd2
|
||||
namespace: discov
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /usr/local/bin/etcd
|
||||
- --name
|
||||
- etcd2
|
||||
- --initial-advertise-peer-urls
|
||||
- http://etcd2:2380
|
||||
- --listen-peer-urls
|
||||
- http://0.0.0.0:2380
|
||||
- --listen-client-urls
|
||||
- http://0.0.0.0:2379
|
||||
- --advertise-client-urls
|
||||
- http://etcd2:2379
|
||||
- --initial-cluster
|
||||
- etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
|
||||
- --initial-cluster-state
|
||||
- new
|
||||
image: quay.io/coreos/etcd:latest
|
||||
name: etcd2
|
||||
ports:
|
||||
- containerPort: 2379
|
||||
name: client
|
||||
protocol: TCP
|
||||
- containerPort: 2380
|
||||
name: server
|
||||
protocol: TCP
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- etcd
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
restartPolicy: Always
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
etcd_node: etcd2
|
||||
name: etcd2
|
||||
namespace: discov
|
||||
spec:
|
||||
ports:
|
||||
- name: client
|
||||
port: 2379
|
||||
protocol: TCP
|
||||
targetPort: 2379
|
||||
- name: server
|
||||
port: 2380
|
||||
protocol: TCP
|
||||
targetPort: 2380
|
||||
selector:
|
||||
etcd_node: etcd2
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
app: etcd
|
||||
etcd_node: etcd3
|
||||
name: etcd3
|
||||
namespace: discov
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /usr/local/bin/etcd
|
||||
- --name
|
||||
- etcd3
|
||||
- --initial-advertise-peer-urls
|
||||
- http://etcd3:2380
|
||||
- --listen-peer-urls
|
||||
- http://0.0.0.0:2380
|
||||
- --listen-client-urls
|
||||
- http://0.0.0.0:2379
|
||||
- --advertise-client-urls
|
||||
- http://etcd3:2379
|
||||
- --initial-cluster
|
||||
- etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
|
||||
- --initial-cluster-state
|
||||
- new
|
||||
image: quay.io/coreos/etcd:latest
|
||||
name: etcd3
|
||||
ports:
|
||||
- containerPort: 2379
|
||||
name: client
|
||||
protocol: TCP
|
||||
- containerPort: 2380
|
||||
name: server
|
||||
protocol: TCP
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- etcd
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
restartPolicy: Always
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
etcd_node: etcd3
|
||||
name: etcd3
|
||||
namespace: discov
|
||||
spec:
|
||||
ports:
|
||||
- name: client
|
||||
port: 2379
|
||||
protocol: TCP
|
||||
targetPort: 2379
|
||||
- name: server
|
||||
port: 2380
|
||||
protocol: TCP
|
||||
targetPort: 2380
|
||||
selector:
|
||||
etcd_node: etcd3
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
app: etcd
|
||||
etcd_node: etcd4
|
||||
name: etcd4
|
||||
namespace: discov
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /usr/local/bin/etcd
|
||||
- --name
|
||||
- etcd4
|
||||
- --initial-advertise-peer-urls
|
||||
- http://etcd4:2380
|
||||
- --listen-peer-urls
|
||||
- http://0.0.0.0:2380
|
||||
- --listen-client-urls
|
||||
- http://0.0.0.0:2379
|
||||
- --advertise-client-urls
|
||||
- http://etcd4:2379
|
||||
- --initial-cluster
|
||||
- etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
|
||||
- --initial-cluster-state
|
||||
- new
|
||||
image: quay.io/coreos/etcd:latest
|
||||
name: etcd4
|
||||
ports:
|
||||
- containerPort: 2379
|
||||
name: client
|
||||
protocol: TCP
|
||||
- containerPort: 2380
|
||||
name: server
|
||||
protocol: TCP
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- etcd
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
restartPolicy: Always
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
etcd_node: etcd4
|
||||
name: etcd4
|
||||
namespace: discov
|
||||
spec:
|
||||
ports:
|
||||
- name: client
|
||||
port: 2379
|
||||
protocol: TCP
|
||||
targetPort: 2379
|
||||
- name: server
|
||||
port: 2380
|
||||
protocol: TCP
|
||||
targetPort: 2380
|
||||
selector:
|
||||
etcd_node: etcd4
|
||||
@@ -12,14 +12,14 @@ func TestBulkExecutor(t *testing.T) {
|
||||
var values []int
|
||||
var lock sync.Mutex
|
||||
|
||||
exeutor := NewBulkExecutor(func(items []interface{}) {
|
||||
executor := NewBulkExecutor(func(items []interface{}) {
|
||||
lock.Lock()
|
||||
values = append(values, len(items))
|
||||
lock.Unlock()
|
||||
}, WithBulkTasks(10), WithBulkInterval(time.Minute))
|
||||
|
||||
for i := 0; i < 50; i++ {
|
||||
exeutor.Add(1)
|
||||
executor.Add(1)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
@@ -40,13 +40,13 @@ func TestBulkExecutorFlushInterval(t *testing.T) {
|
||||
var wait sync.WaitGroup
|
||||
|
||||
wait.Add(1)
|
||||
exeutor := NewBulkExecutor(func(items []interface{}) {
|
||||
executor := NewBulkExecutor(func(items []interface{}) {
|
||||
assert.Equal(t, size, len(items))
|
||||
wait.Done()
|
||||
}, WithBulkTasks(caches), WithBulkInterval(time.Millisecond*100))
|
||||
|
||||
for i := 0; i < size; i++ {
|
||||
exeutor.Add(1)
|
||||
executor.Add(1)
|
||||
}
|
||||
|
||||
wait.Wait()
|
||||
|
||||
@@ -12,14 +12,14 @@ func TestChunkExecutor(t *testing.T) {
|
||||
var values []int
|
||||
var lock sync.Mutex
|
||||
|
||||
exeutor := NewChunkExecutor(func(items []interface{}) {
|
||||
executor := NewChunkExecutor(func(items []interface{}) {
|
||||
lock.Lock()
|
||||
values = append(values, len(items))
|
||||
lock.Unlock()
|
||||
}, WithChunkBytes(10), WithFlushInterval(time.Minute))
|
||||
|
||||
for i := 0; i < 50; i++ {
|
||||
exeutor.Add(1, 1)
|
||||
executor.Add(1, 1)
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
@@ -40,13 +40,13 @@ func TestChunkExecutorFlushInterval(t *testing.T) {
|
||||
var wait sync.WaitGroup
|
||||
|
||||
wait.Add(1)
|
||||
exeutor := NewChunkExecutor(func(items []interface{}) {
|
||||
executor := NewChunkExecutor(func(items []interface{}) {
|
||||
assert.Equal(t, size, len(items))
|
||||
wait.Done()
|
||||
}, WithChunkBytes(caches), WithFlushInterval(time.Millisecond*100))
|
||||
|
||||
for i := 0; i < size; i++ {
|
||||
exeutor.Add(1, 1)
|
||||
executor.Add(1, 1)
|
||||
}
|
||||
|
||||
wait.Wait()
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/proc"
|
||||
"github.com/tal-tech/go-zero/core/syncx"
|
||||
"github.com/tal-tech/go-zero/core/threading"
|
||||
@@ -32,19 +33,21 @@ type (
|
||||
container TaskContainer
|
||||
waitGroup sync.WaitGroup
|
||||
// avoid race condition on waitGroup when calling wg.Add/Done/Wait(...)
|
||||
wgBarrier syncx.Barrier
|
||||
guarded bool
|
||||
newTicker func(duration time.Duration) timex.Ticker
|
||||
lock sync.Mutex
|
||||
wgBarrier syncx.Barrier
|
||||
confirmChan chan lang.PlaceholderType
|
||||
guarded bool
|
||||
newTicker func(duration time.Duration) timex.Ticker
|
||||
lock sync.Mutex
|
||||
}
|
||||
)
|
||||
|
||||
func NewPeriodicalExecutor(interval time.Duration, container TaskContainer) *PeriodicalExecutor {
|
||||
executor := &PeriodicalExecutor{
|
||||
// buffer 1 to let the caller go quickly
|
||||
commander: make(chan interface{}, 1),
|
||||
interval: interval,
|
||||
container: container,
|
||||
commander: make(chan interface{}, 1),
|
||||
interval: interval,
|
||||
container: container,
|
||||
confirmChan: make(chan lang.PlaceholderType),
|
||||
newTicker: func(d time.Duration) timex.Ticker {
|
||||
return timex.NewTicker(interval)
|
||||
},
|
||||
@@ -59,10 +62,12 @@ func NewPeriodicalExecutor(interval time.Duration, container TaskContainer) *Per
|
||||
func (pe *PeriodicalExecutor) Add(task interface{}) {
|
||||
if vals, ok := pe.addAndCheck(task); ok {
|
||||
pe.commander <- vals
|
||||
<-pe.confirmChan
|
||||
}
|
||||
}
|
||||
|
||||
func (pe *PeriodicalExecutor) Flush() bool {
|
||||
pe.enterExecution()
|
||||
return pe.executeTasks(func() interface{} {
|
||||
pe.lock.Lock()
|
||||
defer pe.lock.Unlock()
|
||||
@@ -114,6 +119,8 @@ func (pe *PeriodicalExecutor) backgroundFlush() {
|
||||
select {
|
||||
case vals := <-pe.commander:
|
||||
commanded = true
|
||||
pe.enterExecution()
|
||||
pe.confirmChan <- lang.Placeholder
|
||||
pe.executeTasks(vals)
|
||||
last = timex.Now()
|
||||
case <-ticker.Chan():
|
||||
@@ -135,13 +142,18 @@ func (pe *PeriodicalExecutor) backgroundFlush() {
|
||||
})
|
||||
}
|
||||
|
||||
func (pe *PeriodicalExecutor) executeTasks(tasks interface{}) bool {
|
||||
func (pe *PeriodicalExecutor) doneExecution() {
|
||||
pe.waitGroup.Done()
|
||||
}
|
||||
|
||||
func (pe *PeriodicalExecutor) enterExecution() {
|
||||
pe.wgBarrier.Guard(func() {
|
||||
pe.waitGroup.Add(1)
|
||||
})
|
||||
defer pe.wgBarrier.Guard(func() {
|
||||
pe.waitGroup.Done()
|
||||
})
|
||||
}
|
||||
|
||||
func (pe *PeriodicalExecutor) executeTasks(tasks interface{}) bool {
|
||||
defer pe.doneExecution()
|
||||
|
||||
ok := pe.hasTasks(tasks)
|
||||
if ok {
|
||||
|
||||
@@ -106,6 +106,40 @@ func TestPeriodicalExecutor_Bulk(t *testing.T) {
|
||||
lock.Unlock()
|
||||
}
|
||||
|
||||
func TestPeriodicalExecutor_Wait(t *testing.T) {
|
||||
var lock sync.Mutex
|
||||
executer := NewBulkExecutor(func(tasks []interface{}) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}, WithBulkTasks(1), WithBulkInterval(time.Second))
|
||||
for i := 0; i < 10; i++ {
|
||||
executer.Add(1)
|
||||
}
|
||||
executer.Flush()
|
||||
executer.Wait()
|
||||
}
|
||||
|
||||
func TestPeriodicalExecutor_WaitFast(t *testing.T) {
|
||||
const total = 3
|
||||
var cnt int
|
||||
var lock sync.Mutex
|
||||
executer := NewBulkExecutor(func(tasks []interface{}) {
|
||||
defer func() {
|
||||
cnt++
|
||||
}()
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}, WithBulkTasks(1), WithBulkInterval(10*time.Millisecond))
|
||||
for i := 0; i < total; i++ {
|
||||
executer.Add(2)
|
||||
}
|
||||
executer.Flush()
|
||||
executer.Wait()
|
||||
assert.Equal(t, total, cnt)
|
||||
}
|
||||
|
||||
// go test -benchtime 10s -bench .
|
||||
func BenchmarkExecutor(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
@@ -4,9 +4,8 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/fs"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tal-tech/go-zero/core/fs"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -15,34 +14,34 @@ const (
|
||||
text = `first line
|
||||
Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
|
||||
Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
|
||||
Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
|
||||
Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
|
||||
Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
|
||||
Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
|
||||
Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
|
||||
Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
|
||||
Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
|
||||
Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
|
||||
Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
|
||||
Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
|
||||
` + longLine
|
||||
textWithLastNewline = `first line
|
||||
Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
|
||||
Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
|
||||
Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
|
||||
Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
|
||||
Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
|
||||
Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
|
||||
Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
|
||||
Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
|
||||
Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
|
||||
Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
|
||||
Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
|
||||
Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
|
||||
Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
|
||||
` + longLine + "\n"
|
||||
|
||||
@@ -49,7 +49,7 @@ func From(generate GenerateFunc) Stream {
|
||||
return Range(source)
|
||||
}
|
||||
|
||||
// Just converts the given arbitary items to a Stream.
|
||||
// Just converts the given arbitrary items to a Stream.
|
||||
func Just(items ...interface{}) Stream {
|
||||
source := make(chan interface{}, len(items))
|
||||
for _, item := range items {
|
||||
@@ -195,7 +195,7 @@ func (p Stream) Merge() Stream {
|
||||
return Range(source)
|
||||
}
|
||||
|
||||
// Parallel applies the given ParallenFunc to each item concurrently with given number of workers.
|
||||
// Parallel applies the given ParallelFunc to each item concurrently with given number of workers.
|
||||
func (p Stream) Parallel(fn ParallelFunc, opts ...Option) {
|
||||
p.Walk(func(item interface{}, pipe chan<- interface{}) {
|
||||
fn(item)
|
||||
|
||||
@@ -1,16 +1,8 @@
|
||||
package lang
|
||||
|
||||
import "log"
|
||||
|
||||
var Placeholder PlaceholderType
|
||||
|
||||
type (
|
||||
GenericType = interface{}
|
||||
PlaceholderType = struct{}
|
||||
)
|
||||
|
||||
func Must(err error) {
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
package lang
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMust(t *testing.T) {
|
||||
Must(nil)
|
||||
}
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/iox"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/sysx"
|
||||
"github.com/tal-tech/go-zero/core/timex"
|
||||
)
|
||||
@@ -46,6 +45,7 @@ const (
|
||||
levelInfo = "info"
|
||||
levelError = "error"
|
||||
levelSevere = "severe"
|
||||
levelFatal = "fatal"
|
||||
levelSlow = "slow"
|
||||
levelStat = "stat"
|
||||
|
||||
@@ -100,7 +100,7 @@ type (
|
||||
)
|
||||
|
||||
func MustSetup(c LogConf) {
|
||||
lang.Must(SetUp(c))
|
||||
Must(SetUp(c))
|
||||
}
|
||||
|
||||
// SetUp sets up the logx. If already set up, just return nil.
|
||||
@@ -210,6 +210,15 @@ func Infof(format string, v ...interface{}) {
|
||||
infoSync(fmt.Sprintf(format, v...))
|
||||
}
|
||||
|
||||
func Must(err error) {
|
||||
if err != nil {
|
||||
msg := formatWithCaller(err.Error(), 3)
|
||||
log.Print(msg)
|
||||
output(severeLog, levelFatal, msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func SetLevel(level uint32) {
|
||||
atomic.StoreUint32(&logLevel, level)
|
||||
}
|
||||
|
||||
@@ -131,6 +131,10 @@ func TestSetLevelWithDuration(t *testing.T) {
|
||||
assert.Equal(t, 0, writer.builder.Len())
|
||||
}
|
||||
|
||||
func TestMustNil(t *testing.T) {
|
||||
Must(nil)
|
||||
}
|
||||
|
||||
func BenchmarkCopyByteSliceAppend(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var buf []byte
|
||||
|
||||
44
core/queue/balancedpusher.go
Normal file
44
core/queue/balancedpusher.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
)
|
||||
|
||||
var ErrNoAvailablePusher = errors.New("no available pusher")
|
||||
|
||||
type BalancedPusher struct {
|
||||
name string
|
||||
pushers []Pusher
|
||||
index uint64
|
||||
}
|
||||
|
||||
func NewBalancedPusher(pushers []Pusher) Pusher {
|
||||
return &BalancedPusher{
|
||||
name: generateName(pushers),
|
||||
pushers: pushers,
|
||||
}
|
||||
}
|
||||
|
||||
func (pusher *BalancedPusher) Name() string {
|
||||
return pusher.name
|
||||
}
|
||||
|
||||
func (pusher *BalancedPusher) Push(message string) error {
|
||||
size := len(pusher.pushers)
|
||||
|
||||
for i := 0; i < size; i++ {
|
||||
index := atomic.AddUint64(&pusher.index, 1) % uint64(size)
|
||||
target := pusher.pushers[index]
|
||||
|
||||
if err := target.Push(message); err != nil {
|
||||
logx.Error(err)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return ErrNoAvailablePusher
|
||||
}
|
||||
43
core/queue/balancedpusher_test.go
Normal file
43
core/queue/balancedpusher_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBalancedQueuePusher(t *testing.T) {
|
||||
const numPushers = 100
|
||||
var pushers []Pusher
|
||||
var mockedPushers []*mockedPusher
|
||||
for i := 0; i < numPushers; i++ {
|
||||
p := &mockedPusher{
|
||||
name: "pusher:" + strconv.Itoa(i),
|
||||
}
|
||||
pushers = append(pushers, p)
|
||||
mockedPushers = append(mockedPushers, p)
|
||||
}
|
||||
|
||||
pusher := NewBalancedPusher(pushers)
|
||||
assert.True(t, len(pusher.Name()) > 0)
|
||||
|
||||
for i := 0; i < numPushers*1000; i++ {
|
||||
assert.Nil(t, pusher.Push("item"))
|
||||
}
|
||||
|
||||
var counts []int
|
||||
for _, p := range mockedPushers {
|
||||
counts = append(counts, p.count)
|
||||
}
|
||||
mean := calcMean(counts)
|
||||
variance := calcVariance(mean, counts)
|
||||
assert.True(t, variance < 100, fmt.Sprintf("too big variance - %.2f", variance))
|
||||
}
|
||||
|
||||
func TestBalancedQueuePusher_NoAvailable(t *testing.T) {
|
||||
pusher := NewBalancedPusher(nil)
|
||||
assert.True(t, len(pusher.Name()) == 0)
|
||||
assert.Equal(t, ErrNoAvailablePusher, pusher.Push("item"))
|
||||
}
|
||||
10
core/queue/consumer.go
Normal file
10
core/queue/consumer.go
Normal file
@@ -0,0 +1,10 @@
|
||||
package queue
|
||||
|
||||
type (
|
||||
Consumer interface {
|
||||
Consume(string) error
|
||||
OnEvent(event interface{})
|
||||
}
|
||||
|
||||
ConsumerFactory func() (Consumer, error)
|
||||
)
|
||||
6
core/queue/messagequeue.go
Normal file
6
core/queue/messagequeue.go
Normal file
@@ -0,0 +1,6 @@
|
||||
package queue
|
||||
|
||||
type MessageQueue interface {
|
||||
Start()
|
||||
Stop()
|
||||
}
|
||||
31
core/queue/multipusher.go
Normal file
31
core/queue/multipusher.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package queue
|
||||
|
||||
import "github.com/tal-tech/go-zero/core/errorx"
|
||||
|
||||
type MultiPusher struct {
|
||||
name string
|
||||
pushers []Pusher
|
||||
}
|
||||
|
||||
func NewMultiPusher(pushers []Pusher) Pusher {
|
||||
return &MultiPusher{
|
||||
name: generateName(pushers),
|
||||
pushers: pushers,
|
||||
}
|
||||
}
|
||||
|
||||
func (pusher *MultiPusher) Name() string {
|
||||
return pusher.name
|
||||
}
|
||||
|
||||
func (pusher *MultiPusher) Push(message string) error {
|
||||
var batchError errorx.BatchError
|
||||
|
||||
for _, each := range pusher.pushers {
|
||||
if err := each.Push(message); err != nil {
|
||||
batchError.Add(err)
|
||||
}
|
||||
}
|
||||
|
||||
return batchError.Err()
|
||||
}
|
||||
39
core/queue/multipusher_test.go
Normal file
39
core/queue/multipusher_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMultiQueuePusher(t *testing.T) {
|
||||
const numPushers = 100
|
||||
var pushers []Pusher
|
||||
var mockedPushers []*mockedPusher
|
||||
for i := 0; i < numPushers; i++ {
|
||||
p := &mockedPusher{
|
||||
name: "pusher:" + strconv.Itoa(i),
|
||||
}
|
||||
pushers = append(pushers, p)
|
||||
mockedPushers = append(mockedPushers, p)
|
||||
}
|
||||
|
||||
pusher := NewMultiPusher(pushers)
|
||||
assert.True(t, len(pusher.Name()) > 0)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
_ = pusher.Push("item")
|
||||
}
|
||||
|
||||
var counts []int
|
||||
for _, p := range mockedPushers {
|
||||
counts = append(counts, p.count)
|
||||
}
|
||||
mean := calcMean(counts)
|
||||
variance := calcVariance(mean, counts)
|
||||
assert.True(t, math.Abs(mean-1000*(1-failProba)) < 10)
|
||||
assert.True(t, variance < 100, fmt.Sprintf("too big variance - %.2f", variance))
|
||||
}
|
||||
15
core/queue/producer.go
Normal file
15
core/queue/producer.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package queue
|
||||
|
||||
type (
|
||||
Producer interface {
|
||||
AddListener(listener ProduceListener)
|
||||
Produce() (string, bool)
|
||||
}
|
||||
|
||||
ProduceListener interface {
|
||||
OnProducerPause()
|
||||
OnProducerResume()
|
||||
}
|
||||
|
||||
ProducerFactory func() (Producer, error)
|
||||
)
|
||||
239
core/queue/queue.go
Normal file
239
core/queue/queue.go
Normal file
@@ -0,0 +1,239 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/rescue"
|
||||
"github.com/tal-tech/go-zero/core/stat"
|
||||
"github.com/tal-tech/go-zero/core/threading"
|
||||
"github.com/tal-tech/go-zero/core/timex"
|
||||
)
|
||||
|
||||
const queueName = "queue"
|
||||
|
||||
type (
|
||||
Queue struct {
|
||||
name string
|
||||
metrics *stat.Metrics
|
||||
producerFactory ProducerFactory
|
||||
producerRoutineGroup *threading.RoutineGroup
|
||||
consumerFactory ConsumerFactory
|
||||
consumerRoutineGroup *threading.RoutineGroup
|
||||
producerCount int
|
||||
consumerCount int
|
||||
active int32
|
||||
channel chan string
|
||||
quit chan struct{}
|
||||
listeners []Listener
|
||||
eventLock sync.Mutex
|
||||
eventChannels []chan interface{}
|
||||
}
|
||||
|
||||
Listener interface {
|
||||
OnPause()
|
||||
OnResume()
|
||||
}
|
||||
|
||||
Poller interface {
|
||||
Name() string
|
||||
Poll() string
|
||||
}
|
||||
|
||||
Pusher interface {
|
||||
Name() string
|
||||
Push(string) error
|
||||
}
|
||||
)
|
||||
|
||||
func NewQueue(producerFactory ProducerFactory, consumerFactory ConsumerFactory) *Queue {
|
||||
queue := &Queue{
|
||||
metrics: stat.NewMetrics(queueName),
|
||||
producerFactory: producerFactory,
|
||||
producerRoutineGroup: threading.NewRoutineGroup(),
|
||||
consumerFactory: consumerFactory,
|
||||
consumerRoutineGroup: threading.NewRoutineGroup(),
|
||||
producerCount: runtime.NumCPU(),
|
||||
consumerCount: runtime.NumCPU() << 1,
|
||||
channel: make(chan string),
|
||||
quit: make(chan struct{}),
|
||||
}
|
||||
queue.SetName(queueName)
|
||||
|
||||
return queue
|
||||
}
|
||||
|
||||
func (queue *Queue) AddListener(listener Listener) {
|
||||
queue.listeners = append(queue.listeners, listener)
|
||||
}
|
||||
|
||||
func (queue *Queue) Broadcast(message interface{}) {
|
||||
go func() {
|
||||
queue.eventLock.Lock()
|
||||
defer queue.eventLock.Unlock()
|
||||
|
||||
for _, channel := range queue.eventChannels {
|
||||
channel <- message
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (queue *Queue) SetName(name string) {
|
||||
queue.name = name
|
||||
queue.metrics.SetName(name)
|
||||
}
|
||||
|
||||
func (queue *Queue) SetNumConsumer(count int) {
|
||||
queue.consumerCount = count
|
||||
}
|
||||
|
||||
func (queue *Queue) SetNumProducer(count int) {
|
||||
queue.producerCount = count
|
||||
}
|
||||
|
||||
func (queue *Queue) Start() {
|
||||
queue.startProducers(queue.producerCount)
|
||||
queue.startConsumers(queue.consumerCount)
|
||||
|
||||
queue.producerRoutineGroup.Wait()
|
||||
close(queue.channel)
|
||||
queue.consumerRoutineGroup.Wait()
|
||||
}
|
||||
|
||||
func (queue *Queue) Stop() {
|
||||
close(queue.quit)
|
||||
}
|
||||
|
||||
func (queue *Queue) consume(eventChan chan interface{}) {
|
||||
var consumer Consumer
|
||||
|
||||
for {
|
||||
var err error
|
||||
if consumer, err = queue.consumerFactory(); err != nil {
|
||||
logx.Errorf("Error on creating consumer: %v", err)
|
||||
time.Sleep(time.Second)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case message, ok := <-queue.channel:
|
||||
if ok {
|
||||
queue.consumeOne(consumer, message)
|
||||
} else {
|
||||
logx.Info("Task channel was closed, quitting consumer...")
|
||||
return
|
||||
}
|
||||
case event := <-eventChan:
|
||||
consumer.OnEvent(event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (queue *Queue) consumeOne(consumer Consumer, message string) {
|
||||
threading.RunSafe(func() {
|
||||
startTime := timex.Now()
|
||||
defer func() {
|
||||
duration := timex.Since(startTime)
|
||||
queue.metrics.Add(stat.Task{
|
||||
Duration: duration,
|
||||
})
|
||||
logx.WithDuration(duration).Infof("%s", message)
|
||||
}()
|
||||
|
||||
if err := consumer.Consume(message); err != nil {
|
||||
logx.Errorf("Error occurred while consuming %v: %v", message, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (queue *Queue) pause() {
|
||||
for _, listener := range queue.listeners {
|
||||
listener.OnPause()
|
||||
}
|
||||
}
|
||||
|
||||
func (queue *Queue) produce() {
|
||||
var producer Producer
|
||||
|
||||
for {
|
||||
var err error
|
||||
if producer, err = queue.producerFactory(); err != nil {
|
||||
logx.Errorf("Error on creating producer: %v", err)
|
||||
time.Sleep(time.Second)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
atomic.AddInt32(&queue.active, 1)
|
||||
producer.AddListener(routineListener{
|
||||
queue: queue,
|
||||
})
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-queue.quit:
|
||||
logx.Info("Quitting producer")
|
||||
return
|
||||
default:
|
||||
if v, ok := queue.produceOne(producer); ok {
|
||||
queue.channel <- v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (queue *Queue) produceOne(producer Producer) (string, bool) {
|
||||
// avoid panic quit the producer, just log it and continue
|
||||
defer rescue.Recover()
|
||||
|
||||
return producer.Produce()
|
||||
}
|
||||
|
||||
func (queue *Queue) resume() {
|
||||
for _, listener := range queue.listeners {
|
||||
listener.OnResume()
|
||||
}
|
||||
}
|
||||
|
||||
func (queue *Queue) startConsumers(number int) {
|
||||
for i := 0; i < number; i++ {
|
||||
eventChan := make(chan interface{})
|
||||
queue.eventLock.Lock()
|
||||
queue.eventChannels = append(queue.eventChannels, eventChan)
|
||||
queue.eventLock.Unlock()
|
||||
queue.consumerRoutineGroup.Run(func() {
|
||||
queue.consume(eventChan)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (queue *Queue) startProducers(number int) {
|
||||
for i := 0; i < number; i++ {
|
||||
queue.producerRoutineGroup.Run(func() {
|
||||
queue.produce()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type routineListener struct {
|
||||
queue *Queue
|
||||
}
|
||||
|
||||
func (rl routineListener) OnProducerPause() {
|
||||
if atomic.AddInt32(&rl.queue.active, -1) <= 0 {
|
||||
rl.queue.pause()
|
||||
}
|
||||
}
|
||||
|
||||
func (rl routineListener) OnProducerResume() {
|
||||
if atomic.AddInt32(&rl.queue.active, 1) == 1 {
|
||||
rl.queue.resume()
|
||||
}
|
||||
}
|
||||
94
core/queue/queue_test.go
Normal file
94
core/queue/queue_test.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
consumers = 4
|
||||
rounds = 100
|
||||
)
|
||||
|
||||
func TestQueue(t *testing.T) {
|
||||
producer := newMockedProducer(rounds)
|
||||
consumer := newMockedConsumer()
|
||||
consumer.wait.Add(consumers)
|
||||
q := NewQueue(func() (Producer, error) {
|
||||
return producer, nil
|
||||
}, func() (Consumer, error) {
|
||||
return consumer, nil
|
||||
})
|
||||
q.AddListener(new(mockedListener))
|
||||
q.SetName("mockqueue")
|
||||
q.SetNumConsumer(consumers)
|
||||
q.SetNumProducer(1)
|
||||
q.pause()
|
||||
q.resume()
|
||||
go func() {
|
||||
producer.wait.Wait()
|
||||
q.Stop()
|
||||
}()
|
||||
q.Start()
|
||||
assert.Equal(t, int32(rounds), atomic.LoadInt32(&consumer.count))
|
||||
}
|
||||
|
||||
type mockedConsumer struct {
|
||||
count int32
|
||||
events int32
|
||||
wait sync.WaitGroup
|
||||
}
|
||||
|
||||
func newMockedConsumer() *mockedConsumer {
|
||||
return new(mockedConsumer)
|
||||
}
|
||||
|
||||
func (c *mockedConsumer) Consume(string) error {
|
||||
atomic.AddInt32(&c.count, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *mockedConsumer) OnEvent(interface{}) {
|
||||
if atomic.AddInt32(&c.events, 1) <= consumers {
|
||||
c.wait.Done()
|
||||
}
|
||||
}
|
||||
|
||||
type mockedProducer struct {
|
||||
total int32
|
||||
count int32
|
||||
wait sync.WaitGroup
|
||||
}
|
||||
|
||||
func newMockedProducer(total int32) *mockedProducer {
|
||||
p := new(mockedProducer)
|
||||
p.total = total
|
||||
p.wait.Add(int(total))
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *mockedProducer) AddListener(listener ProduceListener) {
|
||||
}
|
||||
|
||||
func (p *mockedProducer) Produce() (string, bool) {
|
||||
if atomic.AddInt32(&p.count, 1) <= p.total {
|
||||
p.wait.Done()
|
||||
return "item", true
|
||||
} else {
|
||||
time.Sleep(time.Second)
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
type mockedListener struct {
|
||||
}
|
||||
|
||||
func (l *mockedListener) OnPause() {
|
||||
}
|
||||
|
||||
func (l *mockedListener) OnResume() {
|
||||
}
|
||||
12
core/queue/util.go
Normal file
12
core/queue/util.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package queue
|
||||
|
||||
import "strings"
|
||||
|
||||
func generateName(pushers []Pusher) string {
|
||||
names := make([]string, len(pushers))
|
||||
for i, pusher := range pushers {
|
||||
names[i] = pusher.Name()
|
||||
}
|
||||
|
||||
return strings.Join(names, ",")
|
||||
}
|
||||
77
core/queue/util_test.go
Normal file
77
core/queue/util_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/mathx"
|
||||
)
|
||||
|
||||
var (
|
||||
proba = mathx.NewProba()
|
||||
failProba = 0.01
|
||||
)
|
||||
|
||||
func init() {
|
||||
logx.Disable()
|
||||
}
|
||||
|
||||
func TestGenerateName(t *testing.T) {
|
||||
pushers := []Pusher{
|
||||
&mockedPusher{name: "first"},
|
||||
&mockedPusher{name: "second"},
|
||||
&mockedPusher{name: "third"},
|
||||
}
|
||||
|
||||
assert.Equal(t, "first,second,third", generateName(pushers))
|
||||
}
|
||||
|
||||
func TestGenerateNameNil(t *testing.T) {
|
||||
var pushers []Pusher
|
||||
assert.Equal(t, "", generateName(pushers))
|
||||
}
|
||||
|
||||
func calcMean(vals []int) float64 {
|
||||
if len(vals) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var result float64
|
||||
for _, val := range vals {
|
||||
result += float64(val)
|
||||
}
|
||||
return result / float64(len(vals))
|
||||
}
|
||||
|
||||
func calcVariance(mean float64, vals []int) float64 {
|
||||
if len(vals) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var result float64
|
||||
for _, val := range vals {
|
||||
result += math.Pow(float64(val)-mean, 2)
|
||||
}
|
||||
return result / float64(len(vals))
|
||||
}
|
||||
|
||||
type mockedPusher struct {
|
||||
name string
|
||||
count int
|
||||
}
|
||||
|
||||
func (p *mockedPusher) Name() string {
|
||||
return p.name
|
||||
}
|
||||
|
||||
func (p *mockedPusher) Push(s string) error {
|
||||
if proba.TrueOnProba(failProba) {
|
||||
return errors.New("dummy")
|
||||
}
|
||||
|
||||
p.count++
|
||||
return nil
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/tal-tech/go-zero/core/proc"
|
||||
"github.com/tal-tech/go-zero/core/sysx"
|
||||
"github.com/tal-tech/go-zero/core/timex"
|
||||
"github.com/tal-tech/go-zero/core/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -24,7 +23,7 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
reporter = utils.Report
|
||||
reporter func(string)
|
||||
lock sync.RWMutex
|
||||
lessExecutor = executors.NewLessExecutor(time.Minute * 5)
|
||||
dropped int32
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/iox"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -24,17 +24,17 @@ var (
|
||||
|
||||
func init() {
|
||||
cpus, err := perCpuUsage()
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
cores = uint64(len(cpus))
|
||||
|
||||
sets, err := cpuSets()
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
quota = float64(len(sets))
|
||||
cq, err := cpuQuota()
|
||||
if err == nil {
|
||||
if cq != -1 {
|
||||
period, err := cpuPeriod()
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
|
||||
limit := float64(cq) / float64(period)
|
||||
if limit < quota {
|
||||
@@ -44,10 +44,10 @@ func init() {
|
||||
}
|
||||
|
||||
preSystem, err = systemCpuUsage()
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
|
||||
preTotal, err = totalCpuUsage()
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
}
|
||||
|
||||
func RefreshCpu() uint64 {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/collection"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/proc"
|
||||
"github.com/tal-tech/go-zero/core/stat"
|
||||
@@ -33,7 +32,7 @@ type delayTask struct {
|
||||
func init() {
|
||||
var err error
|
||||
timingWheel, err = collection.NewTimingWheel(time.Second, timingWheelSlots, clean)
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
|
||||
proc.AddShutdownListener(func() {
|
||||
timingWheel.Drain(clean)
|
||||
|
||||
@@ -212,10 +212,12 @@ func TestRedis_Persist(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.False(t, ok)
|
||||
err = client.Expire("key", 5)
|
||||
assert.Nil(t, err)
|
||||
ok, err = client.Persist("key")
|
||||
assert.Nil(t, err)
|
||||
assert.True(t, ok)
|
||||
err = client.Expireat("key", time.Now().Unix()+5)
|
||||
assert.Nil(t, err)
|
||||
ok, err = client.Persist("key")
|
||||
assert.Nil(t, err)
|
||||
assert.True(t, ok)
|
||||
@@ -379,7 +381,7 @@ func TestRedis_SortedSet(t *testing.T) {
|
||||
rank, err := client.Zrank("key", "value2")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, int64(1), rank)
|
||||
rank, err = client.Zrank("key", "value4")
|
||||
_, err = client.Zrank("key", "value4")
|
||||
assert.Equal(t, redis.Nil, err)
|
||||
num, err := client.Zrem("key", "value2", "value3")
|
||||
assert.Nil(t, err)
|
||||
|
||||
@@ -249,10 +249,12 @@ func TestRedis_Persist(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.False(t, ok)
|
||||
err = client.Expire("key", 5)
|
||||
assert.Nil(t, err)
|
||||
ok, err = client.Persist("key")
|
||||
assert.Nil(t, err)
|
||||
assert.True(t, ok)
|
||||
err = client.Expireat("key", time.Now().Unix()+5)
|
||||
assert.Nil(t, err)
|
||||
ok, err = client.Persist("key")
|
||||
assert.Nil(t, err)
|
||||
assert.True(t, ok)
|
||||
@@ -447,7 +449,7 @@ func TestRedis_SortedSet(t *testing.T) {
|
||||
rank, err := client.Zrank("key", "value2")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, int64(1), rank)
|
||||
rank, err = client.Zrank("key", "value4")
|
||||
_, err = client.Zrank("key", "value4")
|
||||
assert.Equal(t, Nil, err)
|
||||
num, err := client.Zrem("key", "value2", "value3")
|
||||
assert.Nil(t, err)
|
||||
@@ -558,6 +560,7 @@ func TestRedis_Pipelined(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "1", value)
|
||||
score, err := client.Zscore("zadd", "zadd")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, int64(12), score)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -63,11 +63,6 @@ func (r *replacer) Replace(text string) string {
|
||||
i = j - 1
|
||||
builder.WriteString(r.mapping[string(chars[start:end])])
|
||||
} else {
|
||||
if j < size {
|
||||
end = j + 1
|
||||
} else {
|
||||
end = size
|
||||
}
|
||||
builder.WriteRune(chars[i])
|
||||
}
|
||||
start = -1
|
||||
|
||||
@@ -2,7 +2,11 @@ package stringx
|
||||
|
||||
import "github.com/tal-tech/go-zero/core/lang"
|
||||
|
||||
const defaultMask = '*'
|
||||
|
||||
type (
|
||||
TrieOption func(trie *trieNode)
|
||||
|
||||
Trie interface {
|
||||
Filter(text string) (string, []string, bool)
|
||||
FindKeywords(text string) []string
|
||||
@@ -10,6 +14,7 @@ type (
|
||||
|
||||
trieNode struct {
|
||||
node
|
||||
mask rune
|
||||
}
|
||||
|
||||
scope struct {
|
||||
@@ -18,8 +23,15 @@ type (
|
||||
}
|
||||
)
|
||||
|
||||
func NewTrie(words []string) Trie {
|
||||
func NewTrie(words []string, opts ...TrieOption) Trie {
|
||||
n := new(trieNode)
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(n)
|
||||
}
|
||||
if n.mask == 0 {
|
||||
n.mask = defaultMask
|
||||
}
|
||||
for _, word := range words {
|
||||
n.add(word)
|
||||
}
|
||||
@@ -114,6 +126,12 @@ func (n *trieNode) findKeywordScopes(chars []rune) []scope {
|
||||
|
||||
func (n *trieNode) replaceWithAsterisk(chars []rune, start, stop int) {
|
||||
for i := start; i < stop; i++ {
|
||||
chars[i] = '*'
|
||||
chars[i] = n.mask
|
||||
}
|
||||
}
|
||||
|
||||
func WithMask(mask rune) TrieOption {
|
||||
return func(n *trieNode) {
|
||||
n.mask = mask
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,25 +109,25 @@ func TestTrie(t *testing.T) {
|
||||
func TestTrieSingleWord(t *testing.T) {
|
||||
trie := NewTrie([]string{
|
||||
"闹",
|
||||
})
|
||||
}, WithMask('#'))
|
||||
output, keywords, ok := trie.Filter("今晚真热闹")
|
||||
assert.ElementsMatch(t, []string{"闹"}, keywords)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "今晚真热*", output)
|
||||
assert.Equal(t, "今晚真热#", output)
|
||||
}
|
||||
|
||||
func TestTrieOverlap(t *testing.T) {
|
||||
trie := NewTrie([]string{
|
||||
"一二三四五",
|
||||
"二三四五六七八",
|
||||
})
|
||||
}, WithMask('#'))
|
||||
output, keywords, ok := trie.Filter("零一二三四五六七八九十")
|
||||
assert.ElementsMatch(t, []string{
|
||||
"一二三四五",
|
||||
"二三四五六七八",
|
||||
}, keywords)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "零********九十", output)
|
||||
assert.Equal(t, "零########九十", output)
|
||||
}
|
||||
|
||||
func TestTrieNested(t *testing.T) {
|
||||
@@ -135,7 +135,7 @@ func TestTrieNested(t *testing.T) {
|
||||
"一二三",
|
||||
"一二三四五",
|
||||
"一二三四五六七八",
|
||||
})
|
||||
}, WithMask('#'))
|
||||
output, keywords, ok := trie.Filter("零一二三四五六七八九十")
|
||||
assert.ElementsMatch(t, []string{
|
||||
"一二三",
|
||||
@@ -143,7 +143,7 @@ func TestTrieNested(t *testing.T) {
|
||||
"一二三四五六七八",
|
||||
}, keywords)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "零********九十", output)
|
||||
assert.Equal(t, "零########九十", output)
|
||||
}
|
||||
|
||||
func BenchmarkTrie(b *testing.B) {
|
||||
|
||||
@@ -3,7 +3,7 @@ package sysx
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/stringx"
|
||||
)
|
||||
|
||||
var hostname string
|
||||
@@ -11,7 +11,9 @@ var hostname string
|
||||
func init() {
|
||||
var err error
|
||||
hostname, err = os.Hostname()
|
||||
lang.Must(err)
|
||||
if err != nil {
|
||||
hostname = stringx.RandId()
|
||||
}
|
||||
}
|
||||
|
||||
func Hostname() string {
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
package utils
|
||||
|
||||
func Report(content string) {
|
||||
// TODO: implement the report method
|
||||
}
|
||||
Binary file not shown.
46
doc/goctl.md
46
doc/goctl.md
@@ -3,8 +3,8 @@
|
||||
## goctl用途
|
||||
* 定义api请求
|
||||
* 根据定义的api自动生成golang(后端), java(iOS & Android), typescript(web & 晓程序),dart(flutter)
|
||||
* 生成MySQL CURD (https://goctl.xiaoheiban.cn)
|
||||
* 生成MongoDB CURD (https://goctl.xiaoheiban.cn)
|
||||
* 生成MySQL CURD+Cache
|
||||
* 生成MongoDB CURD+Cache
|
||||
|
||||
## goctl使用说明
|
||||
#### goctl参数说明
|
||||
@@ -179,23 +179,38 @@ service user-api {
|
||||
* 在定义的get/post/put/delete等请求的handler和logic里增加处理业务逻辑的代码
|
||||
|
||||
#### 根据定义好的api文件生成java代码
|
||||
`goctl api java -api user/user.api -dir ./src`
|
||||
```shell
|
||||
goctl api java -api user/user.api -dir ./src
|
||||
```
|
||||
|
||||
#### 根据定义好的api文件生成typescript代码
|
||||
`goctl api ts -api user/user.api -dir ./src -webapi ***`
|
||||
|
||||
ts需要指定webapi所在目录
|
||||
```shell
|
||||
goctl api ts -api user/user.api -dir ./src -webapi ***
|
||||
|
||||
ts需要指定webapi所在目录
|
||||
```
|
||||
|
||||
#### 根据定义好的api文件生成Dart代码
|
||||
`goctl api dart -api user/user.api -dir ./src`
|
||||
```shell
|
||||
goctl api dart -api user/user.api -dir ./src
|
||||
```
|
||||
|
||||
## 根据mysql ddl或者datasource生成model文件
|
||||
|
||||
```shell script
|
||||
$ goctl model mysql -src={filename} -dir={dir} -cache={true|false}
|
||||
```
|
||||
详情参考[model文档](https://github.com/tal-tech/go-zero/blob/master/tools/goctl/model/sql/README.MD)
|
||||
|
||||
## 根据定义好的简单go文件生成mongo代码文件(仅限golang使用)
|
||||
`goctl model mongo -src {{yourDir}}/xiao/service/xhb/user/model/usermodel.go -cache yes`
|
||||
|
||||
-src需要提供简单的usermodel.go文件,里面只需要提供一个结构体即可
|
||||
-cache 控制是否需要缓存 yes=需要 no=不需要
|
||||
src 示例代码如下
|
||||
```
|
||||
```shell
|
||||
goctl model mongo -src {{yourDir}}/xiao/service/xhb/user/model/usermodel.go -cache yes
|
||||
|
||||
-src需要提供简单的usermodel.go文件,里面只需要提供一个结构体即可
|
||||
-cache 控制是否需要缓存 yes=需要 no=不需要
|
||||
src 示例代码如下
|
||||
```
|
||||
```go
|
||||
package model
|
||||
|
||||
type User struct {
|
||||
@@ -210,7 +225,7 @@ type User struct {
|
||||
o是改字段需要生产的操作函数 可以取得get,find,set 分别表示生成返回单个对象的查询方法,返回多个对象的查询方法,设置该字段方法
|
||||
生成的目标文件会覆盖该简单go文件
|
||||
|
||||
## goctl rpc生成
|
||||
## goctl rpc生成(业务剥离中,暂未开放)
|
||||
|
||||
命令 `goctl rpc proto -proto ${proto} -service ${serviceName} -project ${projectName} -dir ${directory} -shared ${shared}`
|
||||
如: `goctl rpc proto -proto test.proto -service test -project xjy -dir .`
|
||||
@@ -261,5 +276,4 @@ type User struct {
|
||||
│ └── test.go [强制覆盖更新]
|
||||
└── test.proto
|
||||
```
|
||||
- 注意 :目前rpc目录生成的proto文件暂不支持import外部proto文件
|
||||
* 如有不理解的地方,随时问Kim/Kevin
|
||||
- 注意 :目前rpc目录生成的proto文件暂不支持import外部proto文件
|
||||
BIN
doc/images/architecture.png
Normal file
BIN
doc/images/architecture.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 333 KiB |
BIN
doc/images/benchmark.png
Normal file
BIN
doc/images/benchmark.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 26 KiB |
BIN
doc/images/trie.png
Normal file
BIN
doc/images/trie.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 91 KiB |
86
doc/keywords.md
Normal file
86
doc/keywords.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# 高效的关键词替换和敏感词过滤工具
|
||||
|
||||
## 1. 算法介绍
|
||||
|
||||
利用高效的Trie树建立关键词树,如下图所示,然后依次查找字符串中的相连字符是否形成树的一条路径
|
||||
|
||||
<img src="images/trie.png" alt="trie" width="350" />
|
||||
|
||||
发现掘金上[这篇文章](https://juejin.im/post/6844903750490914829)写的比较详细,可以一读,具体原理在此不详述。
|
||||
|
||||
## 2. 关键词替换
|
||||
|
||||
支持关键词重叠,自动选用最长的关键词,代码示例如下:
|
||||
|
||||
```go
|
||||
replacer := stringx.NewReplacer(map[string]string{
|
||||
"日本": "法国",
|
||||
"日本的首都": "东京",
|
||||
"东京": "日本的首都",
|
||||
})
|
||||
fmt.Println(replacer.Replace("日本的首都是东京"))
|
||||
```
|
||||
|
||||
可以得到:
|
||||
```
|
||||
东京是日本的首都
|
||||
```
|
||||
|
||||
示例代码见`example/stringx/replace/replace.go`
|
||||
|
||||
## 3. 查找敏感词
|
||||
|
||||
代码示例如下:
|
||||
|
||||
```go
|
||||
filter := stringx.NewTrie([]string{
|
||||
"AV演员",
|
||||
"苍井空",
|
||||
"AV",
|
||||
"日本AV女优",
|
||||
"AV演员色情",
|
||||
})
|
||||
keywords := filter.FindKeywords("日本AV演员兼电视、电影演员。苍井空AV女优是xx出道, 日本AV女优们最精彩的表演是AV演员色情表演")
|
||||
fmt.Println(keywords)
|
||||
```
|
||||
|
||||
可以得到:
|
||||
|
||||
```
|
||||
[苍井空 日本AV女优 AV演员色情 AV AV演员]
|
||||
```
|
||||
|
||||
## 4. 敏感词过滤
|
||||
|
||||
代码示例如下:
|
||||
|
||||
```go
|
||||
filter := stringx.NewTrie([]string{
|
||||
"AV演员",
|
||||
"苍井空",
|
||||
"AV",
|
||||
"日本AV女优",
|
||||
"AV演员色情",
|
||||
}, stringx.WithMask('?')) // 默认替换为*
|
||||
safe, keywords, found := filter.Filter("日本AV演员兼电视、电影演员。苍井空AV女优是xx出道, 日本AV女优们最精彩的表演是AV演员色情表演")
|
||||
fmt.Println(safe)
|
||||
fmt.Println(keywords)
|
||||
fmt.Println(found)
|
||||
```
|
||||
|
||||
可以得到:
|
||||
|
||||
```
|
||||
日本????兼电视、电影演员。?????女优是xx出道, ??????们最精彩的表演是??????表演
|
||||
[苍井空 日本AV女优 AV演员色情 AV AV演员]
|
||||
true
|
||||
```
|
||||
|
||||
示例代码见`example/stringx/filter/filter.go`
|
||||
|
||||
## 5. Benchmark
|
||||
|
||||
| Sentences | Keywords | Regex | go-zero |
|
||||
| --------- | -------- | -------- | ------- |
|
||||
| 10000 | 10000 | 16min10s | 27.2ms |
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/tal-tech/go-zero/core/breaker"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"gopkg.in/cheggaaa/pb.v1"
|
||||
)
|
||||
|
||||
@@ -99,7 +100,7 @@ func main() {
|
||||
|
||||
gb := breaker.NewBreaker()
|
||||
fp, err := os.Create("result.csv")
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
defer fp.Close()
|
||||
fmt.Fprintln(fp, "seconds,state,googleCalls,netflixCalls")
|
||||
|
||||
|
||||
@@ -5,12 +5,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/discov"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
)
|
||||
|
||||
func main() {
|
||||
sub, err := discov.NewSubscriber([]string{"etcd.discovery:2379"}, "028F2C35852D", discov.Exclusive())
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
|
||||
ticker := time.NewTicker(time.Second * 3)
|
||||
defer ticker.Stop()
|
||||
|
||||
@@ -9,12 +9,13 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/threading"
|
||||
"gopkg.in/cheggaaa/pb.v1"
|
||||
)
|
||||
|
||||
var (
|
||||
freq = flag.Int("freq", 100, "frequence")
|
||||
freq = flag.Int("freq", 100, "frequency")
|
||||
duration = flag.String("duration", "10s", "duration")
|
||||
)
|
||||
|
||||
@@ -83,8 +84,8 @@ func (m *metric) reset() counting {
|
||||
return result
|
||||
}
|
||||
|
||||
func runRequests(url string, frequence int, metrics *metric, done <-chan lang.PlaceholderType) {
|
||||
ticker := time.NewTicker(time.Second / time.Duration(frequence))
|
||||
func runRequests(url string, frequency int, metrics *metric, done <-chan lang.PlaceholderType) {
|
||||
ticker := time.NewTicker(time.Second / time.Duration(frequency))
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
@@ -119,14 +120,14 @@ func main() {
|
||||
flag.Parse()
|
||||
|
||||
fp, err := os.Create("result.csv")
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
defer fp.Close()
|
||||
fmt.Fprintln(fp, "seconds,goodOk,goodFail,goodReject,goodErrs,goodUnknowns,goodDropRatio,"+
|
||||
"heavyOk,heavyFail,heavyReject,heavyErrs,heavyUnknowns,heavyDropRatio")
|
||||
|
||||
var gm, hm metric
|
||||
dur, err := time.ParseDuration(*duration)
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
done := make(chan lang.PlaceholderType)
|
||||
group := threading.NewRoutineGroup()
|
||||
group.RunSafe(func() {
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
"github.com/tal-tech/go-zero/core/collection"
|
||||
"github.com/tal-tech/go-zero/core/executors"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/syncx"
|
||||
"gopkg.in/cheggaaa/pb.v1"
|
||||
)
|
||||
@@ -47,7 +47,7 @@ func main() {
|
||||
lessWriter = executors.NewLessExecutor(interval * total / 100)
|
||||
|
||||
fp, err := os.Create("result.csv")
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
defer fp.Close()
|
||||
fmt.Fprintln(fp, "second,maxFlight,flying,agressiveAvgFlying,lazyAvgFlying,bothAvgFlying")
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/fx"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -27,7 +27,7 @@ func main() {
|
||||
flag.Parse()
|
||||
|
||||
fp, err := os.Create("result.csv")
|
||||
lang.Must(err)
|
||||
logx.Must(err)
|
||||
defer fp.Close()
|
||||
fmt.Fprintln(fp, "seconds,total,pass,fail,drop")
|
||||
|
||||
|
||||
@@ -8,11 +8,11 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
exeutor := executors.NewBulkExecutor(func(items []interface{}) {
|
||||
executor := executors.NewBulkExecutor(func(items []interface{}) {
|
||||
fmt.Println(len(items))
|
||||
}, executors.WithBulkTasks(10))
|
||||
for {
|
||||
exeutor.Add(1)
|
||||
executor.Add(1)
|
||||
time.Sleep(time.Millisecond * 90)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/discov"
|
||||
@@ -10,13 +12,31 @@ import (
|
||||
"github.com/tal-tech/go-zero/rpcx"
|
||||
)
|
||||
|
||||
var lb = flag.String("t", "direct", "the load balancer type")
|
||||
|
||||
func main() {
|
||||
cli := rpcx.MustNewClient(rpcx.RpcClientConf{
|
||||
Etcd: discov.EtcdConf{
|
||||
Hosts: []string{"localhost:2379"},
|
||||
Key: "rpcx",
|
||||
},
|
||||
})
|
||||
flag.Parse()
|
||||
|
||||
var cli rpcx.Client
|
||||
switch *lb {
|
||||
case "direct":
|
||||
cli = rpcx.MustNewClient(rpcx.RpcClientConf{
|
||||
Endpoints: []string{
|
||||
"localhost:3456",
|
||||
"localhost:3457",
|
||||
},
|
||||
})
|
||||
case "discov":
|
||||
cli = rpcx.MustNewClient(rpcx.RpcClientConf{
|
||||
Etcd: discov.EtcdConf{
|
||||
Hosts: []string{"localhost:2379"},
|
||||
Key: "rpcx",
|
||||
},
|
||||
})
|
||||
default:
|
||||
log.Fatal("bad load balancing type")
|
||||
}
|
||||
|
||||
greet := unary.NewGreeterClient(cli.Conn())
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
21
example/stringx/filter/filter.go
Normal file
21
example/stringx/filter/filter.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/stringx"
|
||||
)
|
||||
|
||||
func main() {
|
||||
filter := stringx.NewTrie([]string{
|
||||
"AV演员",
|
||||
"苍井空",
|
||||
"AV",
|
||||
"日本AV女优",
|
||||
"AV演员色情",
|
||||
}, stringx.WithMask('?'))
|
||||
safe, keywords, found := filter.Filter("日本AV演员兼电视、电影演员。苍井空AV女优是xx出道, 日本AV女优们最精彩的表演是AV演员色情表演")
|
||||
fmt.Println(safe)
|
||||
fmt.Println(keywords)
|
||||
fmt.Println(found)
|
||||
}
|
||||
16
example/stringx/replace/replace.go
Normal file
16
example/stringx/replace/replace.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/stringx"
|
||||
)
|
||||
|
||||
func main() {
|
||||
replacer := stringx.NewReplacer(map[string]string{
|
||||
"日本": "法国",
|
||||
"日本的首都": "东京",
|
||||
"东京": "日本的首都",
|
||||
})
|
||||
fmt.Println(replacer.Replace("日本的首都是东京"))
|
||||
}
|
||||
3
go.mod
3
go.mod
@@ -13,6 +13,7 @@ require (
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
|
||||
github.com/go-redis/redis v6.15.7+incompatible
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/go-xorm/builder v0.3.4
|
||||
github.com/gogo/protobuf v1.3.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/mock v1.4.3
|
||||
@@ -22,6 +23,7 @@ require (
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect
|
||||
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
|
||||
github.com/justinas/alice v1.2.0
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
@@ -41,6 +43,7 @@ require (
|
||||
github.com/stretchr/testify v1.5.1
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
|
||||
github.com/urfave/cli v1.22.4
|
||||
github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2
|
||||
github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb // indirect
|
||||
go.etcd.io/etcd v0.0.0-20200402134248-51bdeb39e698
|
||||
go.uber.org/automaxprocs v1.3.0
|
||||
|
||||
8
go.sum
8
go.sum
@@ -76,6 +76,10 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-xorm/builder v0.3.4 h1:FxkeGB4Cggdw3tPwutLCpfjng2jugfkg6LDMrd/KsoY=
|
||||
github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw=
|
||||
github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:9wScpmSP5A3Bk8V3XHWUcJmYTh+ZnlHVyc+A4oZYS3Y=
|
||||
github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM=
|
||||
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
@@ -134,6 +138,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtg
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8=
|
||||
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
@@ -266,6 +272,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 h1:YdYsPAZ2pC6Tow/nPZOPQ96O3hm/ToAkGsPLzedXERk=
|
||||
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
|
||||
github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ=
|
||||
github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb h1:ZkM6LRnq40pR1Ox0hTHlnpkcOTuFIDQpZ1IN8rKKhX0=
|
||||
github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
|
||||
|
||||
89
readme.md
89
readme.md
@@ -1,6 +1,27 @@
|
||||
# go-zero项目介绍
|
||||
# go-zero
|
||||
|
||||

|
||||
[](https://github.com/tal-tech/go-zero/actions)
|
||||
[](https://codecov.io/gh/tal-tech/go-zero)
|
||||
[](https://goreportcard.com/report/github.com/tal-tech/go-zero)
|
||||
[](https://github.com/tal-tech/go-zero)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
|
||||
## 0. go-zero介绍
|
||||
|
||||
go-zero是一个集成了各种工程实践的web和rpc框架。通过弹性设计保障了大并发服务端的稳定性,经受了充分的实战检验。
|
||||
|
||||
go-zero 包含极简的 API 定义和生成工具 goctl,可以根据定义的 api 文件一键生成 Go, iOS, Android, Kotlin, Dart, TypeScript, JavaScript 代码,并可直接运行。
|
||||
|
||||
使用go-zero的好处:
|
||||
|
||||
* 轻松获得支撑千万日活服务的稳定性
|
||||
* 内建级联超时控制、限流、自适应熔断、自适应降载等微服务治理能力,无需配置和额外代码
|
||||
* 微服务治理中间件可无缝集成到其它现有框架使用
|
||||
* 极简的API描述,一键生成各端代码
|
||||
* 自动校验客户端请求参数合法性
|
||||
* 大量微服务治理和并发工具包
|
||||
|
||||
<img src="doc/images/architecture.png" alt="架构图" width="1500" />
|
||||
|
||||
## 1. go-zero框架背景
|
||||
|
||||
@@ -53,33 +74,20 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架,有如下
|
||||
|
||||

|
||||
|
||||
## 4. go-zero框架收益
|
||||
|
||||
* 保障大并发服务端的稳定性,经受了充分的实战检验
|
||||
* 极简的API定义
|
||||
* 一键生成Go, iOS, Android, Dart, TypeScript, JavaScript代码,并可直接运行
|
||||
* 服务端自动校验参数合法性
|
||||
|
||||
## 5. go-zero近期开发计划
|
||||
## 4. go-zero近期开发计划
|
||||
|
||||
* 自动生成API mock server,便于客户端开发
|
||||
* 自动生成服务端功能测试
|
||||
|
||||
## 6. Installation
|
||||
## 5. Installation
|
||||
|
||||
1. 在项目目录下通过如下命令安装:
|
||||
在项目目录下通过如下命令安装:
|
||||
|
||||
```shell
|
||||
go get -u github.com/tal-tech/go-zero
|
||||
```
|
||||
```shell
|
||||
go get -u github.com/tal-tech/go-zero
|
||||
```
|
||||
|
||||
2. 代码里导入go-zero
|
||||
|
||||
```go
|
||||
import "github.com/tal-tech/go-zero"
|
||||
```
|
||||
|
||||
## 7. Quick Start
|
||||
## 6. Quick Start
|
||||
|
||||
1. 编译goctl工具
|
||||
|
||||
@@ -93,7 +101,7 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架,有如下
|
||||
|
||||
```go
|
||||
type Request struct {
|
||||
Name string `path:"name"`
|
||||
Name string `path:"name,options=you|me"` // 框架自动验证请求参数是否合法
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
@@ -123,7 +131,6 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架,有如下
|
||||
生成的文件结构如下:
|
||||
|
||||
```
|
||||
.
|
||||
├── greet
|
||||
│ ├── etc
|
||||
│ │ └── greet-api.json // 配置文件
|
||||
@@ -141,30 +148,27 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架,有如下
|
||||
│ └── types
|
||||
│ └── types.go // 请求、返回等类型定义
|
||||
└── greet.api // api描述文件
|
||||
|
||||
8 directories, 9 files
|
||||
```
|
||||
|
||||
生成的代码可以直接运行:
|
||||
|
||||
```shell
|
||||
|
||||
```shell
|
||||
cd greet
|
||||
go run greet.go -f etc/greet-api.json
|
||||
```
|
||||
```
|
||||
|
||||
默认侦听在8888端口(可以在配置文件里修改),可以通过curl请求:
|
||||
默认侦听在8888端口(可以在配置文件里修改),可以通过curl请求:
|
||||
|
||||
```shell
|
||||
```shell
|
||||
➜ go-zero git:(master) curl -w "\ncode: %{http_code}\n" http://localhost:8888/greet/from/kevin
|
||||
{"code":0}
|
||||
code: 200
|
||||
```
|
||||
```
|
||||
|
||||
编写业务代码:
|
||||
编写业务代码:
|
||||
|
||||
* 可以在servicecontext.go里面传递依赖给logic,比如mysql, redis等
|
||||
* 可以在servicecontext.go里面传递依赖给logic,比如mysql, redis等
|
||||
* 在api定义的get/post/put/delete等请求对应的logic里增加业务处理逻辑
|
||||
|
||||
|
||||
4. 可以根据api文件生成前端需要的Java, TypeScript, Dart, JavaScript代码
|
||||
|
||||
```shell
|
||||
@@ -173,6 +177,17 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架,有如下
|
||||
...
|
||||
```
|
||||
|
||||
### 微信交流群
|
||||
## 7. Benchmark
|
||||
|
||||

|
||||
|
||||
[测试代码见这里](https://github.com/smallnest/go-web-framework-benchmark)
|
||||
|
||||
## 8. 文档 (逐步完善中)
|
||||
|
||||
* [goctl使用帮助](doc/goctl.md)
|
||||
* [关键字替换和敏感词过滤工具](doc/keywords.md)
|
||||
|
||||
## 9. 微信交流群
|
||||
|
||||
添加我的微信:kevwan,请注明go-zero,我拉进go-zero社区群🤝
|
||||
|
||||
214
rest/engine.go
Normal file
214
rest/engine.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package rest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/justinas/alice"
|
||||
"github.com/tal-tech/go-zero/core/codec"
|
||||
"github.com/tal-tech/go-zero/core/load"
|
||||
"github.com/tal-tech/go-zero/core/stat"
|
||||
"github.com/tal-tech/go-zero/rest/handler"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
"github.com/tal-tech/go-zero/rest/internal"
|
||||
"github.com/tal-tech/go-zero/rest/router"
|
||||
)
|
||||
|
||||
// use 1000m to represent 100%
|
||||
const topCpuUsage = 1000
|
||||
|
||||
var ErrSignatureConfig = errors.New("bad config for Signature")
|
||||
|
||||
type engine struct {
|
||||
conf RestConf
|
||||
routes []featuredRoutes
|
||||
unauthorizedCallback handler.UnauthorizedCallback
|
||||
unsignedCallback handler.UnsignedCallback
|
||||
middlewares []Middleware
|
||||
shedder load.Shedder
|
||||
priorityShedder load.Shedder
|
||||
}
|
||||
|
||||
func newEngine(c RestConf) *engine {
|
||||
srv := &engine{
|
||||
conf: c,
|
||||
}
|
||||
if c.CpuThreshold > 0 {
|
||||
srv.shedder = load.NewAdaptiveShedder(load.WithCpuThreshold(c.CpuThreshold))
|
||||
srv.priorityShedder = load.NewAdaptiveShedder(load.WithCpuThreshold(
|
||||
(c.CpuThreshold + topCpuUsage) >> 1))
|
||||
}
|
||||
|
||||
return srv
|
||||
}
|
||||
|
||||
func (s *engine) AddRoutes(r featuredRoutes) {
|
||||
s.routes = append(s.routes, r)
|
||||
}
|
||||
|
||||
func (s *engine) SetUnauthorizedCallback(callback handler.UnauthorizedCallback) {
|
||||
s.unauthorizedCallback = callback
|
||||
}
|
||||
|
||||
func (s *engine) SetUnsignedCallback(callback handler.UnsignedCallback) {
|
||||
s.unsignedCallback = callback
|
||||
}
|
||||
|
||||
func (s *engine) Start() error {
|
||||
return s.StartWithRouter(router.NewPatRouter())
|
||||
}
|
||||
|
||||
func (s *engine) StartWithRouter(router httpx.Router) error {
|
||||
if err := s.bindRoutes(router); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return internal.StartHttp(s.conf.Host, s.conf.Port, router)
|
||||
}
|
||||
|
||||
func (s *engine) appendAuthHandler(fr featuredRoutes, chain alice.Chain,
|
||||
verifier func(alice.Chain) alice.Chain) alice.Chain {
|
||||
if fr.jwt.enabled {
|
||||
if len(fr.jwt.prevSecret) == 0 {
|
||||
chain = chain.Append(handler.Authorize(fr.jwt.secret,
|
||||
handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
|
||||
} else {
|
||||
chain = chain.Append(handler.Authorize(fr.jwt.secret,
|
||||
handler.WithPrevSecret(fr.jwt.prevSecret),
|
||||
handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
|
||||
}
|
||||
}
|
||||
|
||||
return verifier(chain)
|
||||
}
|
||||
|
||||
func (s *engine) bindFeaturedRoutes(router httpx.Router, fr featuredRoutes, metrics *stat.Metrics) error {
|
||||
verifier, err := s.signatureVerifier(fr.signature)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, route := range fr.routes {
|
||||
if err := s.bindRoute(fr, router, metrics, route, verifier); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *engine) bindRoute(fr featuredRoutes, router httpx.Router, metrics *stat.Metrics,
|
||||
route Route, verifier func(chain alice.Chain) alice.Chain) error {
|
||||
chain := alice.New(
|
||||
handler.TracingHandler,
|
||||
s.getLogHandler(),
|
||||
handler.MaxConns(s.conf.MaxConns),
|
||||
handler.BreakerHandler(route.Method, route.Path, metrics),
|
||||
handler.SheddingHandler(s.getShedder(fr.priority), metrics),
|
||||
handler.TimeoutHandler(time.Duration(s.conf.Timeout)*time.Millisecond),
|
||||
handler.RecoverHandler,
|
||||
handler.MetricHandler(metrics),
|
||||
handler.PromMetricHandler(route.Path),
|
||||
handler.MaxBytesHandler(s.conf.MaxBytes),
|
||||
handler.GunzipHandler,
|
||||
)
|
||||
chain = s.appendAuthHandler(fr, chain, verifier)
|
||||
|
||||
for _, middleware := range s.middlewares {
|
||||
chain = chain.Append(convertMiddleware(middleware))
|
||||
}
|
||||
handle := chain.ThenFunc(route.Handler)
|
||||
|
||||
return router.Handle(route.Method, route.Path, handle)
|
||||
}
|
||||
|
||||
func (s *engine) bindRoutes(router httpx.Router) error {
|
||||
metrics := s.createMetrics()
|
||||
|
||||
for _, fr := range s.routes {
|
||||
if err := s.bindFeaturedRoutes(router, fr, metrics); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *engine) createMetrics() *stat.Metrics {
|
||||
var metrics *stat.Metrics
|
||||
|
||||
if len(s.conf.Name) > 0 {
|
||||
metrics = stat.NewMetrics(s.conf.Name)
|
||||
} else {
|
||||
metrics = stat.NewMetrics(fmt.Sprintf("%s:%d", s.conf.Host, s.conf.Port))
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
func (s *engine) getLogHandler() func(http.Handler) http.Handler {
|
||||
if s.conf.Verbose {
|
||||
return handler.DetailedLogHandler
|
||||
} else {
|
||||
return handler.LogHandler
|
||||
}
|
||||
}
|
||||
|
||||
func (s *engine) getShedder(priority bool) load.Shedder {
|
||||
if priority && s.priorityShedder != nil {
|
||||
return s.priorityShedder
|
||||
}
|
||||
return s.shedder
|
||||
}
|
||||
|
||||
func (s *engine) signatureVerifier(signature signatureSetting) (func(chain alice.Chain) alice.Chain, error) {
|
||||
if !signature.enabled {
|
||||
return func(chain alice.Chain) alice.Chain {
|
||||
return chain
|
||||
}, nil
|
||||
}
|
||||
|
||||
if len(signature.PrivateKeys) == 0 {
|
||||
if signature.Strict {
|
||||
return nil, ErrSignatureConfig
|
||||
} else {
|
||||
return func(chain alice.Chain) alice.Chain {
|
||||
return chain
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
decrypters := make(map[string]codec.RsaDecrypter)
|
||||
for _, key := range signature.PrivateKeys {
|
||||
fingerprint := key.Fingerprint
|
||||
file := key.KeyFile
|
||||
decrypter, err := codec.NewRsaDecrypter(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
decrypters[fingerprint] = decrypter
|
||||
}
|
||||
|
||||
return func(chain alice.Chain) alice.Chain {
|
||||
if s.unsignedCallback != nil {
|
||||
return chain.Append(handler.ContentSecurityHandler(
|
||||
decrypters, signature.Expiry, signature.Strict, s.unsignedCallback))
|
||||
} else {
|
||||
return chain.Append(handler.ContentSecurityHandler(
|
||||
decrypters, signature.Expiry, signature.Strict))
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *engine) use(middleware Middleware) {
|
||||
s.middlewares = append(s.middlewares, middleware)
|
||||
}
|
||||
|
||||
func convertMiddleware(ware Middleware) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return ware(next.ServeHTTP)
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/dgrijalva/jwt-go"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/rest/internal"
|
||||
"github.com/tal-tech/go-zero/rest/token"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -43,7 +43,7 @@ func Authorize(secret string, opts ...AuthorizeOption) func(http.Handler) http.H
|
||||
opt(&authOpts)
|
||||
}
|
||||
|
||||
parser := internal.NewTokenParser()
|
||||
parser := token.NewTokenParser()
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
token, err := parser.ParseToken(r, secret, authOpts.PrevSecret)
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/tal-tech/go-zero/core/breaker"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/stat"
|
||||
"github.com/tal-tech/go-zero/rest/internal"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
"github.com/tal-tech/go-zero/rest/internal/security"
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ func BreakerHandler(method, path string, metrics *stat.Metrics) func(http.Handle
|
||||
if err != nil {
|
||||
metrics.AddDrop()
|
||||
logx.Errorf("[http] dropped, %s - %s - %s",
|
||||
r.RequestURI, internal.GetRemoteAddr(r), r.UserAgent())
|
||||
r.RequestURI, httpx.GetRemoteAddr(r), r.UserAgent())
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -217,6 +217,7 @@ func TestContentSecurityHandler(t *testing.T) {
|
||||
signature: test.signature,
|
||||
}
|
||||
req, err := buildRequest(setting)
|
||||
assert.Nil(t, err)
|
||||
resp := httptest.NewRecorder()
|
||||
handler.ServeHTTP(resp, req)
|
||||
assert.Equal(t, test.statusCode, resp.Code)
|
||||
@@ -249,6 +250,7 @@ func TestContentSecurityHandler_UnsignedCallback(t *testing.T) {
|
||||
signature: "badone",
|
||||
}
|
||||
req, err := buildRequest(setting)
|
||||
assert.Nil(t, err)
|
||||
resp := httptest.NewRecorder()
|
||||
handler.ServeHTTP(resp, req)
|
||||
assert.Equal(t, http.StatusOK, resp.Code)
|
||||
@@ -285,6 +287,7 @@ func TestContentSecurityHandler_UnsignedCallback_WrongTime(t *testing.T) {
|
||||
fingerprint: fingerprint,
|
||||
}
|
||||
req, err := buildRequest(setting)
|
||||
assert.Nil(t, err)
|
||||
resp := httptest.NewRecorder()
|
||||
handler.ServeHTTP(resp, req)
|
||||
assert.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/timex"
|
||||
"github.com/tal-tech/go-zero/core/utils"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
"github.com/tal-tech/go-zero/rest/internal"
|
||||
)
|
||||
|
||||
@@ -112,10 +113,10 @@ func logBrief(r *http.Request, code int, timer *utils.ElapsedTimer, logs *intern
|
||||
var buf bytes.Buffer
|
||||
duration := timer.Duration()
|
||||
buf.WriteString(fmt.Sprintf("%d - %s - %s - %s - %s",
|
||||
code, r.RequestURI, internal.GetRemoteAddr(r), r.UserAgent(), timex.ReprOfDuration(duration)))
|
||||
code, r.RequestURI, httpx.GetRemoteAddr(r), r.UserAgent(), timex.ReprOfDuration(duration)))
|
||||
if duration > slowThreshold {
|
||||
logx.Slowf("[HTTP] %d - %s - %s - %s - slowcall(%s)",
|
||||
code, r.RequestURI, internal.GetRemoteAddr(r), r.UserAgent(), timex.ReprOfDuration(duration))
|
||||
code, r.RequestURI, httpx.GetRemoteAddr(r), r.UserAgent(), timex.ReprOfDuration(duration))
|
||||
}
|
||||
|
||||
ok := isOkResponse(code)
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/tal-tech/go-zero/core/load"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/core/stat"
|
||||
"github.com/tal-tech/go-zero/rest/internal"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
"github.com/tal-tech/go-zero/rest/internal/security"
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ func SheddingHandler(shedder load.Shedder, metrics *stat.Metrics) func(http.Hand
|
||||
metrics.AddDrop()
|
||||
sheddingStat.IncrementDrop()
|
||||
logx.Errorf("[http] dropped, %s - %s - %s",
|
||||
r.RequestURI, internal.GetRemoteAddr(r), r.UserAgent())
|
||||
r.RequestURI, httpx.GetRemoteAddr(r), r.UserAgent())
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -84,7 +84,6 @@ func ParseHeader(headerValue string) map[string]string {
|
||||
// Parses the post request which contains json in body.
|
||||
func ParseJsonBody(r *http.Request, v interface{}) error {
|
||||
var reader io.Reader
|
||||
|
||||
if withJsonBody(r) {
|
||||
reader = io.LimitReader(r.Body, maxBodyLen)
|
||||
} else {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpx
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -17,6 +18,24 @@ func init() {
|
||||
logx.Disable()
|
||||
}
|
||||
|
||||
func TestError(t *testing.T) {
|
||||
const body = "foo"
|
||||
w := tracedResponseWriter{
|
||||
headers: make(map[string][]string),
|
||||
}
|
||||
Error(&w, errors.New(body))
|
||||
assert.Equal(t, http.StatusBadRequest, w.code)
|
||||
assert.Equal(t, body, strings.TrimSpace(w.builder.String()))
|
||||
}
|
||||
|
||||
func TestOk(t *testing.T) {
|
||||
w := tracedResponseWriter{
|
||||
headers: make(map[string][]string),
|
||||
}
|
||||
Ok(&w)
|
||||
assert.Equal(t, http.StatusOK, w.code)
|
||||
}
|
||||
|
||||
func TestOkJson(t *testing.T) {
|
||||
w := tracedResponseWriter{
|
||||
headers: make(map[string][]string),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package internal
|
||||
package httpx
|
||||
|
||||
import "net/http"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package internal
|
||||
package httpx
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
@@ -16,4 +16,3 @@ func TestGetRemoteAddr(t *testing.T) {
|
||||
r.Header.Set(xForwardFor, host)
|
||||
assert.Equal(t, host, GetRemoteAddr(r))
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
)
|
||||
|
||||
const LogContext = "request_logs"
|
||||
@@ -79,5 +80,5 @@ func formatf(r *http.Request, format string, v ...interface{}) string {
|
||||
}
|
||||
|
||||
func formatWithReq(r *http.Request, v string) string {
|
||||
return fmt.Sprintf("(%s - %s) %s", r.RequestURI, GetRemoteAddr(r), v)
|
||||
return fmt.Sprintf("(%s - %s) %s", r.RequestURI, httpx.GetRemoteAddr(r), v)
|
||||
}
|
||||
|
||||
170
rest/ngin.go
170
rest/ngin.go
@@ -1,170 +0,0 @@
|
||||
package rest
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/rest/handler"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
)
|
||||
|
||||
type (
|
||||
runOptions struct {
|
||||
start func(*engine) error
|
||||
}
|
||||
|
||||
RunOption func(*Server)
|
||||
|
||||
Server struct {
|
||||
ngin *engine
|
||||
opts runOptions
|
||||
}
|
||||
)
|
||||
|
||||
func MustNewServer(c RestConf, opts ...RunOption) *Server {
|
||||
engine, err := NewServer(c, opts...)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
return engine
|
||||
}
|
||||
|
||||
func NewServer(c RestConf, opts ...RunOption) (*Server, error) {
|
||||
if err := c.SetUp(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
server := &Server{
|
||||
ngin: newEngine(c),
|
||||
opts: runOptions{
|
||||
start: func(srv *engine) error {
|
||||
return srv.Start()
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(server)
|
||||
}
|
||||
|
||||
return server, nil
|
||||
}
|
||||
|
||||
func (e *Server) AddRoutes(rs []Route, opts ...RouteOption) {
|
||||
r := featuredRoutes{
|
||||
routes: rs,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
e.ngin.AddRoutes(r)
|
||||
}
|
||||
|
||||
func (e *Server) AddRoute(r Route, opts ...RouteOption) {
|
||||
e.AddRoutes([]Route{r}, opts...)
|
||||
}
|
||||
|
||||
func (e *Server) Start() {
|
||||
handleError(e.opts.start(e.ngin))
|
||||
}
|
||||
|
||||
func (e *Server) Stop() {
|
||||
logx.Close()
|
||||
}
|
||||
|
||||
func (e *Server) Use(middleware Middleware) {
|
||||
e.ngin.use(middleware)
|
||||
}
|
||||
|
||||
func ToMiddleware(handler func(next http.Handler) http.Handler) Middleware {
|
||||
return func(handle http.HandlerFunc) http.HandlerFunc {
|
||||
return handler(handle).ServeHTTP
|
||||
}
|
||||
}
|
||||
|
||||
func WithJwt(secret string) RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
validateSecret(secret)
|
||||
r.jwt.enabled = true
|
||||
r.jwt.secret = secret
|
||||
}
|
||||
}
|
||||
|
||||
func WithJwtTransition(secret, prevSecret string) RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
// why not validate prevSecret, because prevSecret is an already used one,
|
||||
// even it not meet our requirement, we still need to allow the transition.
|
||||
validateSecret(secret)
|
||||
r.jwt.enabled = true
|
||||
r.jwt.secret = secret
|
||||
r.jwt.prevSecret = prevSecret
|
||||
}
|
||||
}
|
||||
|
||||
func WithMiddleware(middleware Middleware, rs ...Route) []Route {
|
||||
routes := make([]Route, len(rs))
|
||||
|
||||
for i := range rs {
|
||||
route := rs[i]
|
||||
routes[i] = Route{
|
||||
Method: route.Method,
|
||||
Path: route.Path,
|
||||
Handler: middleware(route.Handler),
|
||||
}
|
||||
}
|
||||
|
||||
return routes
|
||||
}
|
||||
|
||||
func WithPriority() RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
r.priority = true
|
||||
}
|
||||
}
|
||||
|
||||
func WithRouter(router httpx.Router) RunOption {
|
||||
return func(server *Server) {
|
||||
server.opts.start = func(srv *engine) error {
|
||||
return srv.StartWithRouter(router)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WithSignature(signature SignatureConf) RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
r.signature.enabled = true
|
||||
r.signature.Strict = signature.Strict
|
||||
r.signature.Expiry = signature.Expiry
|
||||
r.signature.PrivateKeys = signature.PrivateKeys
|
||||
}
|
||||
}
|
||||
|
||||
func WithUnauthorizedCallback(callback handler.UnauthorizedCallback) RunOption {
|
||||
return func(engine *Server) {
|
||||
engine.ngin.SetUnauthorizedCallback(callback)
|
||||
}
|
||||
}
|
||||
|
||||
func WithUnsignedCallback(callback handler.UnsignedCallback) RunOption {
|
||||
return func(engine *Server) {
|
||||
engine.ngin.SetUnsignedCallback(callback)
|
||||
}
|
||||
}
|
||||
|
||||
func handleError(err error) {
|
||||
// ErrServerClosed means the server is closed manually
|
||||
if err == nil || err == http.ErrServerClosed {
|
||||
return
|
||||
}
|
||||
|
||||
logx.Error(err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
func validateSecret(secret string) {
|
||||
if len(secret) < 8 {
|
||||
panic("secret's length can't be less than 8")
|
||||
}
|
||||
}
|
||||
316
rest/server.go
316
rest/server.go
@@ -1,214 +1,170 @@
|
||||
package rest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/justinas/alice"
|
||||
"github.com/tal-tech/go-zero/core/codec"
|
||||
"github.com/tal-tech/go-zero/core/load"
|
||||
"github.com/tal-tech/go-zero/core/stat"
|
||||
"github.com/tal-tech/go-zero/core/logx"
|
||||
"github.com/tal-tech/go-zero/rest/handler"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
"github.com/tal-tech/go-zero/rest/internal"
|
||||
"github.com/tal-tech/go-zero/rest/internal/router"
|
||||
)
|
||||
|
||||
// use 1000m to represent 100%
|
||||
const topCpuUsage = 1000
|
||||
|
||||
var ErrSignatureConfig = errors.New("bad config for Signature")
|
||||
|
||||
type engine struct {
|
||||
conf RestConf
|
||||
routes []featuredRoutes
|
||||
unauthorizedCallback handler.UnauthorizedCallback
|
||||
unsignedCallback handler.UnsignedCallback
|
||||
middlewares []Middleware
|
||||
shedder load.Shedder
|
||||
priorityShedder load.Shedder
|
||||
}
|
||||
|
||||
func newEngine(c RestConf) *engine {
|
||||
srv := &engine{
|
||||
conf: c,
|
||||
}
|
||||
if c.CpuThreshold > 0 {
|
||||
srv.shedder = load.NewAdaptiveShedder(load.WithCpuThreshold(c.CpuThreshold))
|
||||
srv.priorityShedder = load.NewAdaptiveShedder(load.WithCpuThreshold(
|
||||
(c.CpuThreshold + topCpuUsage) >> 1))
|
||||
type (
|
||||
runOptions struct {
|
||||
start func(*engine) error
|
||||
}
|
||||
|
||||
return srv
|
||||
}
|
||||
RunOption func(*Server)
|
||||
|
||||
func (s *engine) AddRoutes(r featuredRoutes) {
|
||||
s.routes = append(s.routes, r)
|
||||
}
|
||||
|
||||
func (s *engine) SetUnauthorizedCallback(callback handler.UnauthorizedCallback) {
|
||||
s.unauthorizedCallback = callback
|
||||
}
|
||||
|
||||
func (s *engine) SetUnsignedCallback(callback handler.UnsignedCallback) {
|
||||
s.unsignedCallback = callback
|
||||
}
|
||||
|
||||
func (s *engine) Start() error {
|
||||
return s.StartWithRouter(router.NewPatRouter())
|
||||
}
|
||||
|
||||
func (s *engine) StartWithRouter(router httpx.Router) error {
|
||||
if err := s.bindRoutes(router); err != nil {
|
||||
return err
|
||||
Server struct {
|
||||
ngin *engine
|
||||
opts runOptions
|
||||
}
|
||||
)
|
||||
|
||||
return internal.StartHttp(s.conf.Host, s.conf.Port, router)
|
||||
}
|
||||
|
||||
func (s *engine) appendAuthHandler(fr featuredRoutes, chain alice.Chain,
|
||||
verifier func(alice.Chain) alice.Chain) alice.Chain {
|
||||
if fr.jwt.enabled {
|
||||
if len(fr.jwt.prevSecret) == 0 {
|
||||
chain = chain.Append(handler.Authorize(fr.jwt.secret,
|
||||
handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
|
||||
} else {
|
||||
chain = chain.Append(handler.Authorize(fr.jwt.secret,
|
||||
handler.WithPrevSecret(fr.jwt.prevSecret),
|
||||
handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
|
||||
}
|
||||
}
|
||||
|
||||
return verifier(chain)
|
||||
}
|
||||
|
||||
func (s *engine) bindFeaturedRoutes(router httpx.Router, fr featuredRoutes, metrics *stat.Metrics) error {
|
||||
verifier, err := s.signatureVerifier(fr.signature)
|
||||
func MustNewServer(c RestConf, opts ...RunOption) *Server {
|
||||
engine, err := NewServer(c, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, route := range fr.routes {
|
||||
if err := s.bindRoute(fr, router, metrics, route, verifier); err != nil {
|
||||
return err
|
||||
return engine
|
||||
}
|
||||
|
||||
func NewServer(c RestConf, opts ...RunOption) (*Server, error) {
|
||||
if err := c.SetUp(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
server := &Server{
|
||||
ngin: newEngine(c),
|
||||
opts: runOptions{
|
||||
start: func(srv *engine) error {
|
||||
return srv.Start()
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(server)
|
||||
}
|
||||
|
||||
return server, nil
|
||||
}
|
||||
|
||||
func (e *Server) AddRoutes(rs []Route, opts ...RouteOption) {
|
||||
r := featuredRoutes{
|
||||
routes: rs,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
e.ngin.AddRoutes(r)
|
||||
}
|
||||
|
||||
func (e *Server) AddRoute(r Route, opts ...RouteOption) {
|
||||
e.AddRoutes([]Route{r}, opts...)
|
||||
}
|
||||
|
||||
func (e *Server) Start() {
|
||||
handleError(e.opts.start(e.ngin))
|
||||
}
|
||||
|
||||
func (e *Server) Stop() {
|
||||
logx.Close()
|
||||
}
|
||||
|
||||
func (e *Server) Use(middleware Middleware) {
|
||||
e.ngin.use(middleware)
|
||||
}
|
||||
|
||||
func ToMiddleware(handler func(next http.Handler) http.Handler) Middleware {
|
||||
return func(handle http.HandlerFunc) http.HandlerFunc {
|
||||
return handler(handle).ServeHTTP
|
||||
}
|
||||
}
|
||||
|
||||
func WithJwt(secret string) RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
validateSecret(secret)
|
||||
r.jwt.enabled = true
|
||||
r.jwt.secret = secret
|
||||
}
|
||||
}
|
||||
|
||||
func WithJwtTransition(secret, prevSecret string) RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
// why not validate prevSecret, because prevSecret is an already used one,
|
||||
// even it not meet our requirement, we still need to allow the transition.
|
||||
validateSecret(secret)
|
||||
r.jwt.enabled = true
|
||||
r.jwt.secret = secret
|
||||
r.jwt.prevSecret = prevSecret
|
||||
}
|
||||
}
|
||||
|
||||
func WithMiddleware(middleware Middleware, rs ...Route) []Route {
|
||||
routes := make([]Route, len(rs))
|
||||
|
||||
for i := range rs {
|
||||
route := rs[i]
|
||||
routes[i] = Route{
|
||||
Method: route.Method,
|
||||
Path: route.Path,
|
||||
Handler: middleware(route.Handler),
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return routes
|
||||
}
|
||||
|
||||
func (s *engine) bindRoute(fr featuredRoutes, router httpx.Router, metrics *stat.Metrics,
|
||||
route Route, verifier func(chain alice.Chain) alice.Chain) error {
|
||||
chain := alice.New(
|
||||
handler.TracingHandler,
|
||||
s.getLogHandler(),
|
||||
handler.MaxConns(s.conf.MaxConns),
|
||||
handler.BreakerHandler(route.Method, route.Path, metrics),
|
||||
handler.SheddingHandler(s.getShedder(fr.priority), metrics),
|
||||
handler.TimeoutHandler(time.Duration(s.conf.Timeout)*time.Millisecond),
|
||||
handler.RecoverHandler,
|
||||
handler.MetricHandler(metrics),
|
||||
handler.PromMetricHandler(route.Path),
|
||||
handler.MaxBytesHandler(s.conf.MaxBytes),
|
||||
handler.GunzipHandler,
|
||||
)
|
||||
chain = s.appendAuthHandler(fr, chain, verifier)
|
||||
|
||||
for _, middleware := range s.middlewares {
|
||||
chain = chain.Append(convertMiddleware(middleware))
|
||||
func WithPriority() RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
r.priority = true
|
||||
}
|
||||
handle := chain.ThenFunc(route.Handler)
|
||||
|
||||
return router.Handle(route.Method, route.Path, handle)
|
||||
}
|
||||
|
||||
func (s *engine) bindRoutes(router httpx.Router) error {
|
||||
metrics := s.createMetrics()
|
||||
|
||||
for _, fr := range s.routes {
|
||||
if err := s.bindFeaturedRoutes(router, fr, metrics); err != nil {
|
||||
return err
|
||||
func WithRouter(router httpx.Router) RunOption {
|
||||
return func(server *Server) {
|
||||
server.opts.start = func(srv *engine) error {
|
||||
return srv.StartWithRouter(router)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *engine) createMetrics() *stat.Metrics {
|
||||
var metrics *stat.Metrics
|
||||
|
||||
if len(s.conf.Name) > 0 {
|
||||
metrics = stat.NewMetrics(s.conf.Name)
|
||||
} else {
|
||||
metrics = stat.NewMetrics(fmt.Sprintf("%s:%d", s.conf.Host, s.conf.Port))
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
func (s *engine) getLogHandler() func(http.Handler) http.Handler {
|
||||
if s.conf.Verbose {
|
||||
return handler.DetailedLogHandler
|
||||
} else {
|
||||
return handler.LogHandler
|
||||
func WithSignature(signature SignatureConf) RouteOption {
|
||||
return func(r *featuredRoutes) {
|
||||
r.signature.enabled = true
|
||||
r.signature.Strict = signature.Strict
|
||||
r.signature.Expiry = signature.Expiry
|
||||
r.signature.PrivateKeys = signature.PrivateKeys
|
||||
}
|
||||
}
|
||||
|
||||
func (s *engine) getShedder(priority bool) load.Shedder {
|
||||
if priority && s.priorityShedder != nil {
|
||||
return s.priorityShedder
|
||||
}
|
||||
return s.shedder
|
||||
}
|
||||
|
||||
func (s *engine) signatureVerifier(signature signatureSetting) (func(chain alice.Chain) alice.Chain, error) {
|
||||
if !signature.enabled {
|
||||
return func(chain alice.Chain) alice.Chain {
|
||||
return chain
|
||||
}, nil
|
||||
}
|
||||
|
||||
if len(signature.PrivateKeys) == 0 {
|
||||
if signature.Strict {
|
||||
return nil, ErrSignatureConfig
|
||||
} else {
|
||||
return func(chain alice.Chain) alice.Chain {
|
||||
return chain
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
decrypters := make(map[string]codec.RsaDecrypter)
|
||||
for _, key := range signature.PrivateKeys {
|
||||
fingerprint := key.Fingerprint
|
||||
file := key.KeyFile
|
||||
decrypter, err := codec.NewRsaDecrypter(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
decrypters[fingerprint] = decrypter
|
||||
}
|
||||
|
||||
return func(chain alice.Chain) alice.Chain {
|
||||
if s.unsignedCallback != nil {
|
||||
return chain.Append(handler.ContentSecurityHandler(
|
||||
decrypters, signature.Expiry, signature.Strict, s.unsignedCallback))
|
||||
} else {
|
||||
return chain.Append(handler.ContentSecurityHandler(
|
||||
decrypters, signature.Expiry, signature.Strict))
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *engine) use(middleware Middleware) {
|
||||
s.middlewares = append(s.middlewares, middleware)
|
||||
}
|
||||
|
||||
func convertMiddleware(ware Middleware) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(ware(next.ServeHTTP))
|
||||
func WithUnauthorizedCallback(callback handler.UnauthorizedCallback) RunOption {
|
||||
return func(engine *Server) {
|
||||
engine.ngin.SetUnauthorizedCallback(callback)
|
||||
}
|
||||
}
|
||||
|
||||
func WithUnsignedCallback(callback handler.UnsignedCallback) RunOption {
|
||||
return func(engine *Server) {
|
||||
engine.ngin.SetUnsignedCallback(callback)
|
||||
}
|
||||
}
|
||||
|
||||
func handleError(err error) {
|
||||
// ErrServerClosed means the server is closed manually
|
||||
if err == nil || err == http.ErrServerClosed {
|
||||
return
|
||||
}
|
||||
|
||||
logx.Error(err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
func validateSecret(secret string) {
|
||||
if len(secret) < 8 {
|
||||
panic("secret's length can't be less than 8")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tal-tech/go-zero/rest/httpx"
|
||||
"github.com/tal-tech/go-zero/rest/internal/router"
|
||||
"github.com/tal-tech/go-zero/rest/router"
|
||||
)
|
||||
|
||||
func TestWithMiddleware(t *testing.T) {
|
||||
@@ -1,4 +1,4 @@
|
||||
package internal
|
||||
package token
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
@@ -1,4 +1,4 @@
|
||||
package internal
|
||||
package token
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
@@ -49,10 +49,10 @@ func NewClient(c RpcClientConf, options ...internal.ClientOption) (Client, error
|
||||
|
||||
var client Client
|
||||
var err error
|
||||
if len(c.Server) > 0 {
|
||||
client, err = internal.NewDirectClient(c.Server, opts...)
|
||||
if len(c.Endpoints) > 0 {
|
||||
client, err = internal.NewClient(internal.BuildDirectTarget(c.Endpoints), opts...)
|
||||
} else if err = c.Etcd.Validate(); err == nil {
|
||||
client, err = internal.NewDiscovClient(c.Etcd.Hosts, c.Etcd.Key, opts...)
|
||||
client, err = internal.NewClient(internal.BuildDiscovTarget(c.Etcd.Hosts, c.Etcd.Key), opts...)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -64,7 +64,7 @@ func NewClient(c RpcClientConf, options ...internal.ClientOption) (Client, error
|
||||
}
|
||||
|
||||
func NewClientNoAuth(c discov.EtcdConf) (Client, error) {
|
||||
client, err := internal.NewDiscovClient(c.Hosts, c.Key)
|
||||
client, err := internal.NewClient(internal.BuildDiscovTarget(c.Hosts, c.Key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -74,6 +74,10 @@ func NewClientNoAuth(c discov.EtcdConf) (Client, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewClientWithTarget(target string, opts ...internal.ClientOption) (Client, error) {
|
||||
return internal.NewClient(target, opts...)
|
||||
}
|
||||
|
||||
func (rc *RpcClient) Conn() *grpc.ClientConn {
|
||||
return rc.client.Conn()
|
||||
}
|
||||
|
||||
@@ -21,19 +21,19 @@ type (
|
||||
}
|
||||
|
||||
RpcClientConf struct {
|
||||
Etcd discov.EtcdConf `json:",optional"`
|
||||
Server string `json:",optional=!Etcd"`
|
||||
App string `json:",optional"`
|
||||
Token string `json:",optional"`
|
||||
Timeout int64 `json:",optional"`
|
||||
Etcd discov.EtcdConf `json:",optional"`
|
||||
Endpoints []string `json:",optional=!Etcd"`
|
||||
App string `json:",optional"`
|
||||
Token string `json:",optional"`
|
||||
Timeout int64 `json:",optional"`
|
||||
}
|
||||
)
|
||||
|
||||
func NewDirectClientConf(server, app, token string) RpcClientConf {
|
||||
func NewDirectClientConf(endpoints []string, app, token string) RpcClientConf {
|
||||
return RpcClientConf{
|
||||
Server: server,
|
||||
App: app,
|
||||
Token: token,
|
||||
Endpoints: endpoints,
|
||||
App: app,
|
||||
Token: token,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
62
rpcx/internal/auth/credential_test.go
Normal file
62
rpcx/internal/auth/credential_test.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
func TestParseCredential(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
withNil bool
|
||||
withEmptyMd bool
|
||||
app string
|
||||
token string
|
||||
}{
|
||||
{
|
||||
name: "nil",
|
||||
withNil: true,
|
||||
},
|
||||
{
|
||||
name: "empty md",
|
||||
withEmptyMd: true,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
app: "foo",
|
||||
token: "bar",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var ctx context.Context
|
||||
if test.withNil {
|
||||
ctx = context.Background()
|
||||
} else if test.withEmptyMd {
|
||||
ctx = metadata.NewIncomingContext(context.Background(), metadata.MD{})
|
||||
} else {
|
||||
md := metadata.New(map[string]string{
|
||||
"app": test.app,
|
||||
"token": test.token,
|
||||
})
|
||||
ctx = metadata.NewIncomingContext(context.Background(), md)
|
||||
}
|
||||
cred := ParseCredential(ctx)
|
||||
assert.False(t, cred.RequireTransportSecurity())
|
||||
m, err := cred.GetRequestMetadata(context.Background())
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, test.app, m[appKey])
|
||||
assert.Equal(t, test.token, m[tokenKey])
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
|
||||
const (
|
||||
Name = "p2c_ewma"
|
||||
decayTime = int64(time.Millisecond * 600)
|
||||
decayTime = int64(time.Second * 10) // default value from finagle
|
||||
forcePick = int64(time.Second)
|
||||
initSuccess = 1000
|
||||
throttleSuccess = initSuccess / 2
|
||||
|
||||
@@ -3,7 +3,9 @@ package p2c
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -33,19 +35,31 @@ func TestP2cPicker_Pick(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
candidates int
|
||||
threshold float64
|
||||
}{
|
||||
{
|
||||
name: "single",
|
||||
candidates: 1,
|
||||
threshold: 0.9,
|
||||
},
|
||||
{
|
||||
name: "two",
|
||||
candidates: 2,
|
||||
threshold: 0.5,
|
||||
},
|
||||
{
|
||||
name: "multiple",
|
||||
candidates: 100,
|
||||
threshold: 0.95,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const total = 10000
|
||||
builder := new(p2cPickerBuilder)
|
||||
ready := make(map[resolver.Address]balancer.SubConn)
|
||||
for i := 0; i < test.candidates; i++ {
|
||||
@@ -55,7 +69,9 @@ func TestP2cPicker_Pick(t *testing.T) {
|
||||
}
|
||||
|
||||
picker := builder.Build(ready)
|
||||
for i := 0; i < 10000; i++ {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(total)
|
||||
for i := 0; i < total; i++ {
|
||||
_, done, err := picker.Pick(context.Background(), balancer.PickInfo{
|
||||
FullMethodName: "/",
|
||||
Ctx: context.Background(),
|
||||
@@ -64,11 +80,16 @@ func TestP2cPicker_Pick(t *testing.T) {
|
||||
if i%100 == 0 {
|
||||
err = status.Error(codes.DeadlineExceeded, "deadline")
|
||||
}
|
||||
done(balancer.DoneInfo{
|
||||
Err: err,
|
||||
})
|
||||
go func() {
|
||||
runtime.Gosched()
|
||||
done(balancer.DoneInfo{
|
||||
Err: err,
|
||||
})
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
dist := make(map[interface{}]int)
|
||||
conns := picker.(*p2cPicker).conns
|
||||
for _, conn := range conns {
|
||||
@@ -76,7 +97,8 @@ func TestP2cPicker_Pick(t *testing.T) {
|
||||
}
|
||||
|
||||
entropy := mathx.CalcEntropy(dist)
|
||||
assert.True(t, entropy > .95, fmt.Sprintf("entropy is %f, less than .95", entropy))
|
||||
assert.True(t, entropy > test.threshold, fmt.Sprintf("entropy is %f, less than %f",
|
||||
entropy, test.threshold))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
47
rpcx/internal/chainclientinterceptors_test.go
Normal file
47
rpcx/internal/chainclientinterceptors_test.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func TestWithStreamClientInterceptors(t *testing.T) {
|
||||
opts := WithStreamClientInterceptors()
|
||||
assert.NotNil(t, opts)
|
||||
}
|
||||
|
||||
func TestWithUnaryClientInterceptors(t *testing.T) {
|
||||
opts := WithUnaryClientInterceptors()
|
||||
assert.NotNil(t, opts)
|
||||
}
|
||||
|
||||
func TestChainStreamClientInterceptors_zero(t *testing.T) {
|
||||
interceptors := chainStreamClientInterceptors()
|
||||
_, err := interceptors(context.Background(), nil, new(grpc.ClientConn), "/foo",
|
||||
func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string,
|
||||
opts ...grpc.CallOption) (grpc.ClientStream, error) {
|
||||
return nil, nil
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestChainStreamClientInterceptors_one(t *testing.T) {
|
||||
var called int32
|
||||
interceptors := chainStreamClientInterceptors(func(ctx context.Context, desc *grpc.StreamDesc,
|
||||
cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (
|
||||
grpc.ClientStream, error) {
|
||||
atomic.AddInt32(&called, 1)
|
||||
return nil, nil
|
||||
})
|
||||
_, err := interceptors(context.Background(), nil, new(grpc.ClientConn), "/foo",
|
||||
func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string,
|
||||
opts ...grpc.CallOption) (grpc.ClientStream, error) {
|
||||
return nil, nil
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&called))
|
||||
}
|
||||
@@ -5,12 +5,18 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/tal-tech/go-zero/rpcx/internal/balancer/p2c"
|
||||
"github.com/tal-tech/go-zero/rpcx/internal/clientinterceptors"
|
||||
"github.com/tal-tech/go-zero/rpcx/internal/resolver"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
const dialTimeout = time.Second * 3
|
||||
|
||||
func init() {
|
||||
resolver.RegisterResolver()
|
||||
}
|
||||
|
||||
type (
|
||||
ClientOptions struct {
|
||||
Timeout time.Duration
|
||||
@@ -18,8 +24,26 @@ type (
|
||||
}
|
||||
|
||||
ClientOption func(options *ClientOptions)
|
||||
|
||||
client struct {
|
||||
conn *grpc.ClientConn
|
||||
}
|
||||
)
|
||||
|
||||
func NewClient(target string, opts ...ClientOption) (*client, error) {
|
||||
opts = append(opts, WithDialOption(grpc.WithBalancerName(p2c.Name)))
|
||||
conn, err := dial(target, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &client{conn: conn}, nil
|
||||
}
|
||||
|
||||
func (c *client) Conn() *grpc.ClientConn {
|
||||
return c.conn
|
||||
}
|
||||
|
||||
func WithDialOption(opt grpc.DialOption) ClientOption {
|
||||
return func(options *ClientOptions) {
|
||||
options.DialOptions = append(options.DialOptions, opt)
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package clientinterceptors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tal-tech/go-zero/core/breaker"
|
||||
"github.com/tal-tech/go-zero/core/stat"
|
||||
rcodes "github.com/tal-tech/go-zero/rpcx/internal/codes"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
@@ -49,3 +52,30 @@ func TestBreakerInterceptorDeadlineExceeded(t *testing.T) {
|
||||
assert.True(t, errs[err] > 0)
|
||||
assert.True(t, errs[breaker.ErrServiceUnavailable] > 0)
|
||||
}
|
||||
|
||||
func TestBreakerInterceptor(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "nil",
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
name: "with error",
|
||||
err: errors.New("mock"),
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cc := new(grpc.ClientConn)
|
||||
err := BreakerInterceptor(context.Background(), "/foo", nil, nil, cc,
|
||||
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
|
||||
opts ...grpc.CallOption) error {
|
||||
return test.err
|
||||
})
|
||||
assert.Equal(t, test.err, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
37
rpcx/internal/clientinterceptors/durationinterceptor_test.go
Normal file
37
rpcx/internal/clientinterceptors/durationinterceptor_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package clientinterceptors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func TestDurationInterceptor(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "nil",
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
name: "with error",
|
||||
err: errors.New("mock"),
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cc := new(grpc.ClientConn)
|
||||
err := DurationInterceptor(context.Background(), "/foo", nil, nil, cc,
|
||||
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
|
||||
opts ...grpc.CallOption) error {
|
||||
return test.err
|
||||
})
|
||||
assert.Equal(t, test.err, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package clientinterceptors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func TestPromMetricInterceptor(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "nil",
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
name: "with error",
|
||||
err: errors.New("mock"),
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cc := new(grpc.ClientConn)
|
||||
err := PromMetricInterceptor(context.Background(), "/foo", nil, nil, cc,
|
||||
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
|
||||
opts ...grpc.CallOption) error {
|
||||
return test.err
|
||||
})
|
||||
assert.Equal(t, test.err, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
50
rpcx/internal/clientinterceptors/timeoutinterceptor_test.go
Normal file
50
rpcx/internal/clientinterceptors/timeoutinterceptor_test.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package clientinterceptors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func TestTimeoutInterceptor(t *testing.T) {
|
||||
timeouts := []time.Duration{0, time.Millisecond * 10}
|
||||
for _, timeout := range timeouts {
|
||||
t.Run(strconv.FormatInt(int64(timeout), 10), func(t *testing.T) {
|
||||
interceptor := TimeoutInterceptor(timeout)
|
||||
cc := new(grpc.ClientConn)
|
||||
err := interceptor(context.Background(), "/foo", nil, nil, cc,
|
||||
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
|
||||
opts ...grpc.CallOption) error {
|
||||
return nil
|
||||
},
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutInterceptor_timeout(t *testing.T) {
|
||||
const timeout = time.Millisecond * 10
|
||||
interceptor := TimeoutInterceptor(timeout)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
cc := new(grpc.ClientConn)
|
||||
err := interceptor(ctx, "/foo", nil, nil, cc,
|
||||
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
|
||||
opts ...grpc.CallOption) error {
|
||||
defer wg.Done()
|
||||
tm, ok := ctx.Deadline()
|
||||
assert.True(t, ok)
|
||||
assert.True(t, tm.Before(time.Now().Add(timeout+time.Millisecond)))
|
||||
return nil
|
||||
})
|
||||
wg.Wait()
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
53
rpcx/internal/clientinterceptors/tracinginterceptor_test.go
Normal file
53
rpcx/internal/clientinterceptors/tracinginterceptor_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package clientinterceptors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tal-tech/go-zero/core/trace"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
func TestTracingInterceptor(t *testing.T) {
|
||||
var run int32
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
cc := new(grpc.ClientConn)
|
||||
err := TracingInterceptor(context.Background(), "/foo", nil, nil, cc,
|
||||
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
|
||||
opts ...grpc.CallOption) error {
|
||||
defer wg.Done()
|
||||
atomic.AddInt32(&run, 1)
|
||||
return nil
|
||||
})
|
||||
wg.Wait()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&run))
|
||||
}
|
||||
|
||||
func TestTracingInterceptor_GrpcFormat(t *testing.T) {
|
||||
var run int32
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
md := metadata.New(map[string]string{
|
||||
"foo": "bar",
|
||||
})
|
||||
carrier, err := trace.Inject(trace.GrpcFormat, md)
|
||||
assert.Nil(t, err)
|
||||
ctx, _ := trace.StartServerSpan(context.Background(), carrier, "user", "/foo")
|
||||
cc := new(grpc.ClientConn)
|
||||
err = TracingInterceptor(ctx, "/foo", nil, nil, cc,
|
||||
func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
|
||||
opts ...grpc.CallOption) error {
|
||||
defer wg.Done()
|
||||
atomic.AddInt32(&run, 1)
|
||||
return nil
|
||||
})
|
||||
wg.Wait()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&run))
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/balancer/roundrobin"
|
||||
)
|
||||
|
||||
type DirectClient struct {
|
||||
conn *grpc.ClientConn
|
||||
}
|
||||
|
||||
func NewDirectClient(server string, opts ...ClientOption) (*DirectClient, error) {
|
||||
opts = append(opts, WithDialOption(grpc.WithBalancerName(roundrobin.Name)))
|
||||
conn, err := dial(server, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &DirectClient{
|
||||
conn: conn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *DirectClient) Conn() *grpc.ClientConn {
|
||||
return c.conn
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/tal-tech/go-zero/rpcx/internal/balancer/p2c"
|
||||
"github.com/tal-tech/go-zero/rpcx/internal/resolver"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func init() {
|
||||
resolver.RegisterResolver()
|
||||
}
|
||||
|
||||
type DiscovClient struct {
|
||||
conn *grpc.ClientConn
|
||||
}
|
||||
|
||||
func NewDiscovClient(endpoints []string, key string, opts ...ClientOption) (*DiscovClient, error) {
|
||||
opts = append(opts, WithDialOption(grpc.WithBalancerName(p2c.Name)))
|
||||
target := fmt.Sprintf("%s://%s/%s", resolver.DiscovScheme,
|
||||
strings.Join(endpoints, resolver.EndpointSep), key)
|
||||
conn, err := dial(target, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &DiscovClient{conn: conn}, nil
|
||||
}
|
||||
|
||||
func (c *DiscovClient) Conn() *grpc.ClientConn {
|
||||
return c.conn
|
||||
}
|
||||
32
rpcx/internal/resolver/directbuilder.go
Normal file
32
rpcx/internal/resolver/directbuilder.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
type directBuilder struct{}
|
||||
|
||||
func (d *directBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (
|
||||
resolver.Resolver, error) {
|
||||
var addrs []resolver.Address
|
||||
endpoints := strings.FieldsFunc(target.Endpoint, func(r rune) bool {
|
||||
return r == EndpointSepChar
|
||||
})
|
||||
|
||||
for _, val := range subset(endpoints, subsetSize) {
|
||||
addrs = append(addrs, resolver.Address{
|
||||
Addr: val,
|
||||
})
|
||||
}
|
||||
cc.UpdateState(resolver.State{
|
||||
Addresses: addrs,
|
||||
})
|
||||
|
||||
return &nopResolver{cc: cc}, nil
|
||||
}
|
||||
|
||||
func (d *directBuilder) Scheme() string {
|
||||
return DirectScheme
|
||||
}
|
||||
52
rpcx/internal/resolver/directbuilder_test.go
Normal file
52
rpcx/internal/resolver/directbuilder_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tal-tech/go-zero/core/lang"
|
||||
"github.com/tal-tech/go-zero/core/mathx"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
func TestDirectBuilder_Build(t *testing.T) {
|
||||
tests := []int{
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
subsetSize / 2,
|
||||
subsetSize,
|
||||
subsetSize * 2,
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(strconv.Itoa(test), func(t *testing.T) {
|
||||
var servers []string
|
||||
for i := 0; i < test; i++ {
|
||||
servers = append(servers, fmt.Sprintf("localhost:%d", i))
|
||||
}
|
||||
var b directBuilder
|
||||
cc := new(mockedClientConn)
|
||||
_, err := b.Build(resolver.Target{
|
||||
Scheme: DirectScheme,
|
||||
Endpoint: strings.Join(servers, ","),
|
||||
}, cc, resolver.BuildOptions{})
|
||||
assert.Nil(t, err)
|
||||
size := mathx.MinInt(test, subsetSize)
|
||||
assert.Equal(t, size, len(cc.state.Addresses))
|
||||
m := make(map[string]lang.PlaceholderType)
|
||||
for _, each := range cc.state.Addresses {
|
||||
m[each.Addr] = lang.Placeholder
|
||||
}
|
||||
assert.Equal(t, size, len(m))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectBuilder_Scheme(t *testing.T) {
|
||||
var b directBuilder
|
||||
assert.Equal(t, DirectScheme, b.Scheme())
|
||||
}
|
||||
41
rpcx/internal/resolver/discovbuilder.go
Normal file
41
rpcx/internal/resolver/discovbuilder.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/discov"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
type discovBuilder struct{}
|
||||
|
||||
func (d *discovBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (
|
||||
resolver.Resolver, error) {
|
||||
hosts := strings.FieldsFunc(target.Authority, func(r rune) bool {
|
||||
return r == EndpointSepChar
|
||||
})
|
||||
sub, err := discov.NewSubscriber(hosts, target.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
update := func() {
|
||||
var addrs []resolver.Address
|
||||
for _, val := range subset(sub.Values(), subsetSize) {
|
||||
addrs = append(addrs, resolver.Address{
|
||||
Addr: val,
|
||||
})
|
||||
}
|
||||
cc.UpdateState(resolver.State{
|
||||
Addresses: addrs,
|
||||
})
|
||||
}
|
||||
sub.AddListener(update)
|
||||
update()
|
||||
|
||||
return &nopResolver{cc: cc}, nil
|
||||
}
|
||||
|
||||
func (d *discovBuilder) Scheme() string {
|
||||
return DiscovScheme
|
||||
}
|
||||
@@ -2,67 +2,34 @@ package resolver
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/tal-tech/go-zero/core/discov"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
const (
|
||||
DiscovScheme = "discov"
|
||||
EndpointSep = ","
|
||||
subsetSize = 32
|
||||
DirectScheme = "direct"
|
||||
DiscovScheme = "discov"
|
||||
EndpointSepChar = ','
|
||||
subsetSize = 32
|
||||
)
|
||||
|
||||
var builder discovBuilder
|
||||
var (
|
||||
EndpointSep = fmt.Sprintf("%c", EndpointSepChar)
|
||||
dirBuilder directBuilder
|
||||
disBuilder discovBuilder
|
||||
)
|
||||
|
||||
type discovBuilder struct{}
|
||||
|
||||
func (b *discovBuilder) Scheme() string {
|
||||
return DiscovScheme
|
||||
func RegisterResolver() {
|
||||
resolver.Register(&dirBuilder)
|
||||
resolver.Register(&disBuilder)
|
||||
}
|
||||
|
||||
func (b *discovBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (
|
||||
resolver.Resolver, error) {
|
||||
if target.Scheme != DiscovScheme {
|
||||
return nil, fmt.Errorf("bad scheme: %s", target.Scheme)
|
||||
}
|
||||
|
||||
hosts := strings.Split(target.Authority, EndpointSep)
|
||||
sub, err := discov.NewSubscriber(hosts, target.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
update := func() {
|
||||
var addrs []resolver.Address
|
||||
for _, val := range subset(sub.Values(), subsetSize) {
|
||||
addrs = append(addrs, resolver.Address{
|
||||
Addr: val,
|
||||
})
|
||||
}
|
||||
cc.UpdateState(resolver.State{
|
||||
Addresses: addrs,
|
||||
})
|
||||
}
|
||||
sub.AddListener(update)
|
||||
update()
|
||||
|
||||
return &discovResolver{
|
||||
cc: cc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type discovResolver struct {
|
||||
type nopResolver struct {
|
||||
cc resolver.ClientConn
|
||||
}
|
||||
|
||||
func (r *discovResolver) Close() {
|
||||
func (r *nopResolver) Close() {
|
||||
}
|
||||
|
||||
func (r *discovResolver) ResolveNow(options resolver.ResolveNowOptions) {
|
||||
}
|
||||
|
||||
func RegisterResolver() {
|
||||
resolver.Register(&builder)
|
||||
func (r *nopResolver) ResolveNow(options resolver.ResolveNowOptions) {
|
||||
}
|
||||
|
||||
36
rpcx/internal/resolver/resolver_test.go
Normal file
36
rpcx/internal/resolver/resolver_test.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc/resolver"
|
||||
"google.golang.org/grpc/serviceconfig"
|
||||
)
|
||||
|
||||
func TestNopResolver(t *testing.T) {
|
||||
// make sure ResolveNow & Close don't panic
|
||||
var r nopResolver
|
||||
r.ResolveNow(resolver.ResolveNowOptions{})
|
||||
r.Close()
|
||||
}
|
||||
|
||||
type mockedClientConn struct {
|
||||
state resolver.State
|
||||
}
|
||||
|
||||
func (m *mockedClientConn) UpdateState(state resolver.State) {
|
||||
m.state = state
|
||||
}
|
||||
|
||||
func (m *mockedClientConn) ReportError(err error) {
|
||||
}
|
||||
|
||||
func (m *mockedClientConn) NewAddress(addresses []resolver.Address) {
|
||||
}
|
||||
|
||||
func (m *mockedClientConn) NewServiceConfig(serviceConfig string) {
|
||||
}
|
||||
|
||||
func (m *mockedClientConn) ParseServiceConfig(serviceConfigJSON string) *serviceconfig.ParseResult {
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user