remove no need empty line (#29)

* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* trim no need line

Co-authored-by: kingxt <dream4kingxt@163.com>

.codecov.yml

@@ -0,0 +1,4 @@
+ignore:
+  - "doc"
+  - "example"
+  - "tools"

.github/workflows/go.yml

@@ -27,4 +27,9 @@ jobs:
         go get -v -t -d ./...
 
     - name: Test
-      run: go test -v -race ./...
+      run: go test -race -coverprofile=coverage.txt -covermode=atomic ./...
+
+    - name: Codecov
+      uses: codecov/codecov-action@v1.0.6
+      with:
+        token: ${{secrets.CODECOV_TOKEN}}

.gitlab-ci.yml

@@ -1,18 +0,0 @@
-stages:
-- analysis
-
-variables:
-  GOPATH: '/runner-cache/zero'
-  GOCACHE: '/runner-cache/zero'
-  GOPROXY: 'https://goproxy.cn,direct'
-
-analysis:
-  stage: analysis
-  image: golang
-  script:
-    - go version && go env
-    - go test -short $(go list ./...) | grep -v "no test"
-  only:
-   - merge_requests
-  tags:
-    - common

.golangci.yml

@@ -1,36 +0,0 @@
-run:
-  #  concurrency: 6
-  timeout: 5m
-  skip-dirs:
-    - core
-    - doc
-    - example
-    - rest
-    - rpcx
-    - tools
-
-
-linters:
-  disable-all: true
-  enable:
-    - bodyclose
-    - deadcode
-    - errcheck
-    - gosimple
-    - govet
-    - ineffassign
-    - staticcheck
-    - structcheck
-    - typecheck
-    - unused
-    - varcheck
-#    - dupl
-
-
-linters-settings:
-
-issues:
-  exclude-rules:
-    - linters:
-        - staticcheck
-      text: 'SA1019: (baseresponse.BoolResponse|oldresponse.FormatBadRequestResponse|oldresponse.FormatResponse)|SA5008: unknown JSON option ("optional"|"default=|"range=|"options=)'

core/breaker/breakers.go

@@ -60,17 +60,15 @@ func do(name string, execute func(b Breaker) error) error {
 	lock.RUnlock()
 	if ok {
 		return execute(b)
-	} else {
-		lock.Lock()
-		b, ok = breakers[name]
-		if ok {
-			lock.Unlock()
-			return execute(b)
-		} else {
-			b = NewBreaker(WithName(name))
-			breakers[name] = b
-			lock.Unlock()
-			return execute(b)
-		}
 	}
+
+	lock.Lock()
+	b, ok = breakers[name]
+	if !ok {
+		b = NewBreaker(WithName(name))
+		breakers[name] = b
+	}
+	lock.Unlock()
+
+	return execute(b)
 }

core/collection/cache_test.go

@@ -72,9 +72,9 @@ func TestCacheWithLruEvicts(t *testing.T) {
 	cache.Set("third", "third element")
 	cache.Set("fourth", "fourth element")
 
-	value, ok := cache.Get("first")
+	_, ok := cache.Get("first")
 	assert.False(t, ok)
-	value, ok = cache.Get("second")
+	value, ok := cache.Get("second")
 	assert.True(t, ok)
 	assert.Equal(t, "second element", value)
 	value, ok = cache.Get("third")
@@ -94,9 +94,9 @@ func TestCacheWithLruEvicted(t *testing.T) {
 	cache.Set("third", "third element")
 	cache.Set("fourth", "fourth element")
 
-	value, ok := cache.Get("first")
+	_, ok := cache.Get("first")
 	assert.False(t, ok)
-	value, ok = cache.Get("second")
+	value, ok := cache.Get("second")
 	assert.True(t, ok)
 	assert.Equal(t, "second element", value)
 	cache.Set("fifth", "fifth element")

core/collection/timingwheel_test.go

@@ -213,7 +213,10 @@ func TestTimingWheel_SetTimer(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		test := test
 		t.Run(stringx.RandId(), func(t *testing.T) {
+			t.Parallel()
+
 			var count int32
 			ticker := timex.NewFakeTicker()
 			tick := func() {
@@ -291,7 +294,10 @@ func TestTimingWheel_SetAndMoveThenStart(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		test := test
 		t.Run(stringx.RandId(), func(t *testing.T) {
+			t.Parallel()
+
 			var count int32
 			ticker := timex.NewFakeTicker()
 			tick := func() {
@@ -376,7 +382,10 @@ func TestTimingWheel_SetAndMoveTwice(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		test := test
 		t.Run(stringx.RandId(), func(t *testing.T) {
+			t.Parallel()
+
 			var count int32
 			ticker := timex.NewFakeTicker()
 			tick := func() {
@@ -454,7 +463,10 @@ func TestTimingWheel_ElapsedAndSet(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		test := test
 		t.Run(stringx.RandId(), func(t *testing.T) {
+			t.Parallel()
+
 			var count int32
 			ticker := timex.NewFakeTicker()
 			tick := func() {
@@ -542,7 +554,10 @@ func TestTimingWheel_ElapsedAndSetThenMove(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		test := test
 		t.Run(stringx.RandId(), func(t *testing.T) {
+			t.Parallel()
+
 			var count int32
 			ticker := timex.NewFakeTicker()
 			tick := func() {

core/contextx/valueonlycontext_test.go

@@ -34,7 +34,7 @@ func TestContextCancel(t *testing.T) {
 	assert.NotEqual(t, context.Canceled, c2.Err())
 }
 
-func TestConextDeadline(t *testing.T) {
+func TestContextDeadline(t *testing.T) {
 	c, _ := context.WithDeadline(context.Background(), time.Now().Add(10*time.Millisecond))
 	o := ValueOnlyFrom(c)
 	select {

core/discov/facade.go

@@ -2,7 +2,7 @@ package discov
 
 import (
 	"github.com/tal-tech/go-zero/core/discov/internal"
-	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
 )
 
 type (
@@ -26,7 +26,7 @@ func NewFacade(endpoints []string) Facade {
 
 func (f Facade) Client() internal.EtcdClient {
 	conn, err := f.registry.GetConn(f.endpoints)
-	lang.Must(err)
+	logx.Must(err)
 	return conn
 }
 

core/discov/kubernetes/discov-namespace.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: discovery
+  name: discov

core/discov/kubernetes/etcd.yaml

@@ -1,16 +1,16 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: discov
-  namespace: discovery
+  name: etcd
+  namespace: discov
 spec:
   ports:
-    - name: discov-port
+    - name: etcd-port
       port: 2379
       protocol: TCP
       targetPort: 2379
   selector:
-    app: discov
+    app: etcd
 
 ---
 
@@ -18,30 +18,30 @@ apiVersion: v1
 kind: Pod
 metadata:
   labels:
-    app: discov
-    discov_node: discov0
-  name: discov0
-  namespace: discovery
+    app: etcd
+    etcd_node: etcd0
+  name: etcd0
+  namespace: discov
 spec:
   containers:
     - command:
         - /usr/local/bin/etcd
         - --name
-        - discov0
+        - etcd0
         - --initial-advertise-peer-urls
-        - http://discov0:2380
+        - http://etcd0:2380
         - --listen-peer-urls
         - http://0.0.0.0:2380
         - --listen-client-urls
         - http://0.0.0.0:2379
         - --advertise-client-urls
-        - http://discov0:2379
+        - http://etcd0:2379
         - --initial-cluster
-        - discov0=http://discov0:2380,discov1=http://discov1:2380,discov2=http://discov2:2380,discov3=http://discov3:2380,discov4=http://discov4:2380
+        - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
-      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/etcd:latest
-      name: discov0
+      image: quay.io/coreos/etcd:latest
+      name: etcd0
       ports:
         - containerPort: 2379
           name: client
@@ -49,8 +49,6 @@ spec:
         - containerPort: 2380
           name: server
           protocol: TCP
-  imagePullSecrets:
-    - name: aliyun
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -59,7 +57,7 @@ spec:
               - key: app
                 operator: In
                 values:
-                  - discov
+                  - etcd
           topologyKey: "kubernetes.io/hostname"
   restartPolicy: Always
 
@@ -69,9 +67,9 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    discov_node: discov0
-  name: discov0
-  namespace: discovery
+    etcd_node: etcd0
+  name: etcd0
+  namespace: discov
 spec:
   ports:
     - name: client
@@ -83,7 +81,7 @@ spec:
       protocol: TCP
       targetPort: 2380
   selector:
-    discov_node: discov0
+    etcd_node: etcd0
 
 ---
 
@@ -91,30 +89,30 @@ apiVersion: v1
 kind: Pod
 metadata:
   labels:
-    app: discov
-    discov_node: discov1
-  name: discov1
-  namespace: discovery
+    app: etcd
+    etcd_node: etcd1
+  name: etcd1
+  namespace: discov
 spec:
   containers:
     - command:
         - /usr/local/bin/etcd
         - --name
-        - discov1
+        - etcd1
         - --initial-advertise-peer-urls
-        - http://discov1:2380
+        - http://etcd1:2380
         - --listen-peer-urls
         - http://0.0.0.0:2380
         - --listen-client-urls
         - http://0.0.0.0:2379
         - --advertise-client-urls
-        - http://discov1:2379
+        - http://etcd1:2379
         - --initial-cluster
-        - discov0=http://discov0:2380,discov1=http://discov1:2380,discov2=http://discov2:2380,discov3=http://discov3:2380,discov4=http://discov4:2380
+        - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
-      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/etcd:latest
-      name: discov1
+      image: quay.io/coreos/etcd:latest
+      name: etcd1
       ports:
         - containerPort: 2379
           name: client
@@ -122,8 +120,6 @@ spec:
         - containerPort: 2380
           name: server
           protocol: TCP
-  imagePullSecrets:
-    - name: aliyun
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -132,7 +128,7 @@ spec:
               - key: app
                 operator: In
                 values:
-                  - discov
+                  - etcd
           topologyKey: "kubernetes.io/hostname"
   restartPolicy: Always
 
@@ -142,9 +138,9 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    discov_node: discov1
-  name: discov1
-  namespace: discovery
+    etcd_node: etcd1
+  name: etcd1
+  namespace: discov
 spec:
   ports:
     - name: client
@@ -156,7 +152,7 @@ spec:
       protocol: TCP
       targetPort: 2380
   selector:
-    discov_node: discov1
+    etcd_node: etcd1
 
 ---
 
@@ -164,30 +160,30 @@ apiVersion: v1
 kind: Pod
 metadata:
   labels:
-    app: discov
-    discov_node: discov2
-  name: discov2
-  namespace: discovery
+    app: etcd
+    etcd_node: etcd2
+  name: etcd2
+  namespace: discov
 spec:
   containers:
     - command:
         - /usr/local/bin/etcd
         - --name
-        - discov2
+        - etcd2
         - --initial-advertise-peer-urls
-        - http://discov2:2380
+        - http://etcd2:2380
         - --listen-peer-urls
         - http://0.0.0.0:2380
         - --listen-client-urls
         - http://0.0.0.0:2379
         - --advertise-client-urls
-        - http://discov2:2379
+        - http://etcd2:2379
         - --initial-cluster
-        - discov0=http://discov0:2380,discov1=http://discov1:2380,discov2=http://discov2:2380,discov3=http://discov3:2380,discov4=http://discov4:2380
+        - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
-      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/etcd:latest
-      name: discov2
+      image: quay.io/coreos/etcd:latest
+      name: etcd2
       ports:
         - containerPort: 2379
           name: client
@@ -195,8 +191,6 @@ spec:
         - containerPort: 2380
           name: server
           protocol: TCP
-  imagePullSecrets:
-    - name: aliyun
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -205,7 +199,7 @@ spec:
               - key: app
                 operator: In
                 values:
-                  - discov
+                  - etcd
           topologyKey: "kubernetes.io/hostname"
   restartPolicy: Always
 
@@ -215,9 +209,9 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    discov_node: discov2
-  name: discov2
-  namespace: discovery
+    etcd_node: etcd2
+  name: etcd2
+  namespace: discov
 spec:
   ports:
     - name: client
@@ -229,7 +223,7 @@ spec:
       protocol: TCP
       targetPort: 2380
   selector:
-    discov_node: discov2
+    etcd_node: etcd2
 
 ---
 
@@ -237,30 +231,30 @@ apiVersion: v1
 kind: Pod
 metadata:
   labels:
-    app: discov
-    discov_node: discov3
-  name: discov3
-  namespace: discovery
+    app: etcd
+    etcd_node: etcd3
+  name: etcd3
+  namespace: discov
 spec:
   containers:
     - command:
         - /usr/local/bin/etcd
         - --name
-        - discov3
+        - etcd3
         - --initial-advertise-peer-urls
-        - http://discov3:2380
+        - http://etcd3:2380
         - --listen-peer-urls
         - http://0.0.0.0:2380
         - --listen-client-urls
         - http://0.0.0.0:2379
         - --advertise-client-urls
-        - http://discov3:2379
+        - http://etcd3:2379
         - --initial-cluster
-        - discov0=http://discov0:2380,discov1=http://discov1:2380,discov2=http://discov2:2380,discov3=http://discov3:2380,discov4=http://discov4:2380
+        - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
-      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/etcd:latest
-      name: discov3
+      image: quay.io/coreos/etcd:latest
+      name: etcd3
       ports:
         - containerPort: 2379
           name: client
@@ -268,8 +262,6 @@ spec:
         - containerPort: 2380
           name: server
           protocol: TCP
-  imagePullSecrets:
-    - name: aliyun
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -278,7 +270,7 @@ spec:
               - key: app
                 operator: In
                 values:
-                  - discov
+                  - etcd
           topologyKey: "kubernetes.io/hostname"
   restartPolicy: Always
 
@@ -288,9 +280,9 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    discov_node: discov3
-  name: discov3
-  namespace: discovery
+    etcd_node: etcd3
+  name: etcd3
+  namespace: discov
 spec:
   ports:
     - name: client
@@ -302,7 +294,7 @@ spec:
       protocol: TCP
       targetPort: 2380
   selector:
-    discov_node: discov3
+    etcd_node: etcd3
 
 ---
 
@@ -310,30 +302,30 @@ apiVersion: v1
 kind: Pod
 metadata:
   labels:
-    app: discov
-    discov_node: discov4
-  name: discov4
-  namespace: discovery
+    app: etcd
+    etcd_node: etcd4
+  name: etcd4
+  namespace: discov
 spec:
   containers:
     - command:
         - /usr/local/bin/etcd
         - --name
-        - discov4
+        - etcd4
         - --initial-advertise-peer-urls
-        - http://discov4:2380
+        - http://etcd4:2380
         - --listen-peer-urls
         - http://0.0.0.0:2380
         - --listen-client-urls
         - http://0.0.0.0:2379
         - --advertise-client-urls
-        - http://discov4:2379
+        - http://etcd4:2379
         - --initial-cluster
-        - discov0=http://discov0:2380,discov1=http://discov1:2380,discov2=http://discov2:2380,discov3=http://discov3:2380,discov4=http://discov4:2380
+        - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
-      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/etcd:latest
-      name: discov4
+      image: quay.io/coreos/etcd:latest
+      name: etcd4
       ports:
         - containerPort: 2379
           name: client
@@ -341,8 +333,6 @@ spec:
         - containerPort: 2380
           name: server
           protocol: TCP
-  imagePullSecrets:
-    - name: aliyun
   affinity:
     podAntiAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -351,7 +341,7 @@ spec:
               - key: app
                 operator: In
                 values:
-                  - discov
+                  - etcd
           topologyKey: "kubernetes.io/hostname"
   restartPolicy: Always
 
@@ -361,9 +351,9 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    discov_node: discov4
-  name: discov4
-  namespace: discovery
+    etcd_node: etcd4
+  name: etcd4
+  namespace: discov
 spec:
   ports:
     - name: client
@@ -375,4 +365,4 @@ spec:
       protocol: TCP
       targetPort: 2380
   selector:
-    discov_node: discov4
+    etcd_node: etcd4

core/executors/bulkexecutor_test.go

@@ -12,14 +12,14 @@ func TestBulkExecutor(t *testing.T) {
 	var values []int
 	var lock sync.Mutex
 
-	exeutor := NewBulkExecutor(func(items []interface{}) {
+	executor := NewBulkExecutor(func(items []interface{}) {
 		lock.Lock()
 		values = append(values, len(items))
 		lock.Unlock()
 	}, WithBulkTasks(10), WithBulkInterval(time.Minute))
 
 	for i := 0; i < 50; i++ {
-		exeutor.Add(1)
+		executor.Add(1)
 		time.Sleep(time.Millisecond)
 	}
 
@@ -40,13 +40,13 @@ func TestBulkExecutorFlushInterval(t *testing.T) {
 	var wait sync.WaitGroup
 
 	wait.Add(1)
-	exeutor := NewBulkExecutor(func(items []interface{}) {
+	executor := NewBulkExecutor(func(items []interface{}) {
 		assert.Equal(t, size, len(items))
 		wait.Done()
 	}, WithBulkTasks(caches), WithBulkInterval(time.Millisecond*100))
 
 	for i := 0; i < size; i++ {
-		exeutor.Add(1)
+		executor.Add(1)
 	}
 
 	wait.Wait()

core/executors/chunkexecutor_test.go

@@ -12,14 +12,14 @@ func TestChunkExecutor(t *testing.T) {
 	var values []int
 	var lock sync.Mutex
 
-	exeutor := NewChunkExecutor(func(items []interface{}) {
+	executor := NewChunkExecutor(func(items []interface{}) {
 		lock.Lock()
 		values = append(values, len(items))
 		lock.Unlock()
 	}, WithChunkBytes(10), WithFlushInterval(time.Minute))
 
 	for i := 0; i < 50; i++ {
-		exeutor.Add(1, 1)
+		executor.Add(1, 1)
 		time.Sleep(time.Millisecond)
 	}
 
@@ -40,13 +40,13 @@ func TestChunkExecutorFlushInterval(t *testing.T) {
 	var wait sync.WaitGroup
 
 	wait.Add(1)
-	exeutor := NewChunkExecutor(func(items []interface{}) {
+	executor := NewChunkExecutor(func(items []interface{}) {
 		assert.Equal(t, size, len(items))
 		wait.Done()
 	}, WithChunkBytes(caches), WithFlushInterval(time.Millisecond*100))
 
 	for i := 0; i < size; i++ {
-		exeutor.Add(1, 1)
+		executor.Add(1, 1)
 	}
 
 	wait.Wait()

core/executors/periodicalexecutor.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/core/proc"
 	"github.com/tal-tech/go-zero/core/syncx"
 	"github.com/tal-tech/go-zero/core/threading"
@@ -32,19 +33,21 @@ type (
 		container TaskContainer
 		waitGroup sync.WaitGroup
 		// avoid race condition on waitGroup when calling wg.Add/Done/Wait(...)
-		wgBarrier syncx.Barrier
-		guarded   bool
-		newTicker func(duration time.Duration) timex.Ticker
-		lock      sync.Mutex
+		wgBarrier   syncx.Barrier
+		confirmChan chan lang.PlaceholderType
+		guarded     bool
+		newTicker   func(duration time.Duration) timex.Ticker
+		lock        sync.Mutex
 	}
 )
 
 func NewPeriodicalExecutor(interval time.Duration, container TaskContainer) *PeriodicalExecutor {
 	executor := &PeriodicalExecutor{
 		// buffer 1 to let the caller go quickly
-		commander: make(chan interface{}, 1),
-		interval:  interval,
-		container: container,
+		commander:   make(chan interface{}, 1),
+		interval:    interval,
+		container:   container,
+		confirmChan: make(chan lang.PlaceholderType),
 		newTicker: func(d time.Duration) timex.Ticker {
 			return timex.NewTicker(interval)
 		},
@@ -59,10 +62,12 @@ func NewPeriodicalExecutor(interval time.Duration, container TaskContainer) *Per
 func (pe *PeriodicalExecutor) Add(task interface{}) {
 	if vals, ok := pe.addAndCheck(task); ok {
 		pe.commander <- vals
+		<-pe.confirmChan
 	}
 }
 
 func (pe *PeriodicalExecutor) Flush() bool {
+	pe.enterExecution()
 	return pe.executeTasks(func() interface{} {
 		pe.lock.Lock()
 		defer pe.lock.Unlock()
@@ -114,6 +119,8 @@ func (pe *PeriodicalExecutor) backgroundFlush() {
 			select {
 			case vals := <-pe.commander:
 				commanded = true
+				pe.enterExecution()
+				pe.confirmChan <- lang.Placeholder
 				pe.executeTasks(vals)
 				last = timex.Now()
 			case <-ticker.Chan():
@@ -135,13 +142,18 @@ func (pe *PeriodicalExecutor) backgroundFlush() {
 	})
 }
 
-func (pe *PeriodicalExecutor) executeTasks(tasks interface{}) bool {
+func (pe *PeriodicalExecutor) doneExecution() {
+	pe.waitGroup.Done()
+}
+
+func (pe *PeriodicalExecutor) enterExecution() {
 	pe.wgBarrier.Guard(func() {
 		pe.waitGroup.Add(1)
 	})
-	defer pe.wgBarrier.Guard(func() {
-		pe.waitGroup.Done()
-	})
+}
+
+func (pe *PeriodicalExecutor) executeTasks(tasks interface{}) bool {
+	defer pe.doneExecution()
 
 	ok := pe.hasTasks(tasks)
 	if ok {

core/executors/periodicalexecutor_test.go

@@ -106,6 +106,40 @@ func TestPeriodicalExecutor_Bulk(t *testing.T) {
 	lock.Unlock()
 }
 
+func TestPeriodicalExecutor_Wait(t *testing.T) {
+	var lock sync.Mutex
+	executer := NewBulkExecutor(func(tasks []interface{}) {
+		lock.Lock()
+		defer lock.Unlock()
+		time.Sleep(10 * time.Millisecond)
+	}, WithBulkTasks(1), WithBulkInterval(time.Second))
+	for i := 0; i < 10; i++ {
+		executer.Add(1)
+	}
+	executer.Flush()
+	executer.Wait()
+}
+
+func TestPeriodicalExecutor_WaitFast(t *testing.T) {
+	const total = 3
+	var cnt int
+	var lock sync.Mutex
+	executer := NewBulkExecutor(func(tasks []interface{}) {
+		defer func() {
+			cnt++
+		}()
+		lock.Lock()
+		defer lock.Unlock()
+		time.Sleep(10 * time.Millisecond)
+	}, WithBulkTasks(1), WithBulkInterval(10*time.Millisecond))
+	for i := 0; i < total; i++ {
+		executer.Add(2)
+	}
+	executer.Flush()
+	executer.Wait()
+	assert.Equal(t, total, cnt)
+}
+
 // go test -benchtime 10s -bench .
 func BenchmarkExecutor(b *testing.B) {
 	b.ReportAllocs()

core/filex/file_test.go

@@ -4,9 +4,8 @@ import (
 	"os"
 	"testing"
 
-	"github.com/tal-tech/go-zero/core/fs"
-
 	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/fs"
 )
 
 const (
@@ -15,34 +14,34 @@ const (
 	text          = `first line
 Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
 Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
-Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
+Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
 Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
 Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
 Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
 Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
-Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
+Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
 Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
 Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
 Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
 Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
-Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
+Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
 Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
 Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
 ` + longLine
 	textWithLastNewline = `first line
 Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
 Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
-Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
+Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
 Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
 Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
 Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
 Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
-Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
+Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
 Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
 Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
 Cum sociis natoque penatibus et magnis dis parturient. Phasellus laoreet lorem vel dolor tempus vehicula. Vivamus sagittis lacus vel augue laoreet rutrum faucibus. Integer legentibus erat a ante historiarum dapibus.
 Quisque ut dolor gravida, placerat libero vel, euismod. Quam temere in vitiis, legem sancimus haerentia. Qui ipsorum lingua Celtae, nostra Galli appellantur. Quis aute iure reprehenderit in voluptate velit esse. Fabio vel iudice vincam, sunt in culpa qui officia. Cras mattis iudicium purus sit amet fermentum.
-Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculus sed magna.
+Quo usque tandem abutere, Catilina, patientia nostra? Gallia est omnis divisa in partes tres, quarum. Quam diu etiam furor iste tuus nos eludet? Quid securi etiam tamquam eu fugiat nulla pariatur. Curabitur blandit tempus ardua ridiculous sed magna.
 Magna pars studiorum, prodita quaerimus. Cum ceteris in veneratione tui montes, nascetur mus. Morbi odio eros, volutpat ut pharetra vitae, lobortis sed nibh. Plura mihi bona sunt, inclinet, amari petere vellent. Idque Caesaris facere voluntate liceret: sese habere. Tu quoque, Brute, fili mi, nihil timor populi, nihil!
 Tityre, tu patulae recubans sub tegmine fagi dolor. Inmensae subtilitatis, obscuris et malesuada fames. Quae vero auctorem tractata ab fiducia dicuntur.
 ` + longLine + "\n"

core/fx/fn.go

@@ -49,7 +49,7 @@ func From(generate GenerateFunc) Stream {
 	return Range(source)
 }
 
-// Just converts the given arbitary items to a Stream.
+// Just converts the given arbitrary items to a Stream.
 func Just(items ...interface{}) Stream {
 	source := make(chan interface{}, len(items))
 	for _, item := range items {
@@ -195,7 +195,7 @@ func (p Stream) Merge() Stream {
 	return Range(source)
 }
 
-// Parallel applies the given ParallenFunc to each item concurrently with given number of workers.
+// Parallel applies the given ParallelFunc to each item concurrently with given number of workers.
 func (p Stream) Parallel(fn ParallelFunc, opts ...Option) {
 	p.Walk(func(item interface{}, pipe chan<- interface{}) {
 		fn(item)

core/lang/lang.go

@@ -1,16 +1,8 @@
 package lang
 
-import "log"
-
 var Placeholder PlaceholderType
 
 type (
 	GenericType     = interface{}
 	PlaceholderType = struct{}
 )
-
-func Must(err error) {
-	if err != nil {
-		log.Fatal(err)
-	}
-}

core/lang/lang_test.go

@@ -1,7 +0,0 @@
-package lang
-
-import "testing"
-
-func TestMust(t *testing.T) {
-	Must(nil)
-}

core/logx/logs.go

@@ -17,7 +17,6 @@ import (
 	"sync/atomic"
 
 	"github.com/tal-tech/go-zero/core/iox"
-	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/core/sysx"
 	"github.com/tal-tech/go-zero/core/timex"
 )
@@ -46,6 +45,7 @@ const (
 	levelInfo   = "info"
 	levelError  = "error"
 	levelSevere = "severe"
+	levelFatal  = "fatal"
 	levelSlow   = "slow"
 	levelStat   = "stat"
 
@@ -100,7 +100,7 @@ type (
 )
 
 func MustSetup(c LogConf) {
-	lang.Must(SetUp(c))
+	Must(SetUp(c))
 }
 
 // SetUp sets up the logx. If already set up, just return nil.
@@ -210,6 +210,15 @@ func Infof(format string, v ...interface{}) {
 	infoSync(fmt.Sprintf(format, v...))
 }
 
+func Must(err error) {
+	if err != nil {
+		msg := formatWithCaller(err.Error(), 3)
+		log.Print(msg)
+		output(severeLog, levelFatal, msg)
+		os.Exit(1)
+	}
+}
+
 func SetLevel(level uint32) {
 	atomic.StoreUint32(&logLevel, level)
 }

core/logx/logs_test.go

@@ -131,6 +131,10 @@ func TestSetLevelWithDuration(t *testing.T) {
 	assert.Equal(t, 0, writer.builder.Len())
 }
 
+func TestMustNil(t *testing.T) {
+	Must(nil)
+}
+
 func BenchmarkCopyByteSliceAppend(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		var buf []byte

core/queue/balancedpusher.go

@@ -9,24 +9,24 @@ import (
 
 var ErrNoAvailablePusher = errors.New("no available pusher")
 
-type BalancedQueuePusher struct {
+type BalancedPusher struct {
 	name    string
 	pushers []Pusher
 	index   uint64
 }
 
-func NewBalancedQueuePusher(pushers []Pusher) Pusher {
-	return &BalancedQueuePusher{
+func NewBalancedPusher(pushers []Pusher) Pusher {
+	return &BalancedPusher{
 		name:    generateName(pushers),
 		pushers: pushers,
 	}
 }
 
-func (pusher *BalancedQueuePusher) Name() string {
+func (pusher *BalancedPusher) Name() string {
 	return pusher.name
 }
 
-func (pusher *BalancedQueuePusher) Push(message string) error {
+func (pusher *BalancedPusher) Push(message string) error {
 	size := len(pusher.pushers)
 
 	for i := 0; i < size; i++ {

core/queue/balancedpusher_test.go

@@ -20,7 +20,7 @@ func TestBalancedQueuePusher(t *testing.T) {
 		mockedPushers = append(mockedPushers, p)
 	}
 
-	pusher := NewBalancedQueuePusher(pushers)
+	pusher := NewBalancedPusher(pushers)
 	assert.True(t, len(pusher.Name()) > 0)
 
 	for i := 0; i < numPushers*1000; i++ {
@@ -37,7 +37,7 @@ func TestBalancedQueuePusher(t *testing.T) {
 }
 
 func TestBalancedQueuePusher_NoAvailable(t *testing.T) {
-	pusher := NewBalancedQueuePusher(nil)
+	pusher := NewBalancedPusher(nil)
 	assert.True(t, len(pusher.Name()) == 0)
 	assert.Equal(t, ErrNoAvailablePusher, pusher.Push("item"))
 }

core/queue/multipusher.go

@@ -2,23 +2,23 @@ package queue
 
 import "github.com/tal-tech/go-zero/core/errorx"
 
-type MultiQueuePusher struct {
+type MultiPusher struct {
 	name    string
 	pushers []Pusher
 }
 
-func NewMultiQueuePusher(pushers []Pusher) Pusher {
-	return &MultiQueuePusher{
+func NewMultiPusher(pushers []Pusher) Pusher {
+	return &MultiPusher{
 		name:    generateName(pushers),
 		pushers: pushers,
 	}
 }
 
-func (pusher *MultiQueuePusher) Name() string {
+func (pusher *MultiPusher) Name() string {
 	return pusher.name
 }
 
-func (pusher *MultiQueuePusher) Push(message string) error {
+func (pusher *MultiPusher) Push(message string) error {
 	var batchError errorx.BatchError
 
 	for _, each := range pusher.pushers {

core/queue/multipusher_test.go

@@ -21,7 +21,7 @@ func TestMultiQueuePusher(t *testing.T) {
 		mockedPushers = append(mockedPushers, p)
 	}
 
-	pusher := NewMultiQueuePusher(pushers)
+	pusher := NewMultiPusher(pushers)
 	assert.True(t, len(pusher.Name()) > 0)
 
 	for i := 0; i < 1000; i++ {

core/stat/alert.go

@@ -14,7 +14,6 @@ import (
 	"github.com/tal-tech/go-zero/core/proc"
 	"github.com/tal-tech/go-zero/core/sysx"
 	"github.com/tal-tech/go-zero/core/timex"
-	"github.com/tal-tech/go-zero/core/utils"
 )
 
 const (
@@ -24,7 +23,7 @@ const (
 )
 
 var (
-	reporter     = utils.Report
+	reporter     func(string)
 	lock         sync.RWMutex
 	lessExecutor = executors.NewLessExecutor(time.Minute * 5)
 	dropped      int32

core/stat/internal/cpu_linux.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/iox"
-	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
 )
 
 const (
@@ -22,19 +22,30 @@ var (
 	cores     uint64
 )
 
+// if /proc not present, ignore the cpu calcuation, like wsl linux
 func init() {
 	cpus, err := perCpuUsage()
-	lang.Must(err)
-	cores = uint64(len(cpus))
+	if err != nil {
+		logx.Error(err)
+		return
+	}
 
+	cores = uint64(len(cpus))
 	sets, err := cpuSets()
-	lang.Must(err)
+	if err != nil {
+		logx.Error(err)
+		return
+	}
+
 	quota = float64(len(sets))
 	cq, err := cpuQuota()
 	if err == nil {
 		if cq != -1 {
 			period, err := cpuPeriod()
-			lang.Must(err)
+			if err != nil {
+				logx.Error(err)
+				return
+			}
 
 			limit := float64(cq) / float64(period)
 			if limit < quota {
@@ -44,10 +55,16 @@ func init() {
 	}
 
 	preSystem, err = systemCpuUsage()
-	lang.Must(err)
+	if err != nil {
+		logx.Error(err)
+		return
+	}
 
 	preTotal, err = totalCpuUsage()
-	lang.Must(err)
+	if err != nil {
+		logx.Error(err)
+		return
+	}
 }
 
 func RefreshCpu() uint64 {

core/stores/internal/cleaner.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/collection"
-	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/proc"
 	"github.com/tal-tech/go-zero/core/stat"
@@ -33,7 +32,7 @@ type delayTask struct {
 func init() {
 	var err error
 	timingWheel, err = collection.NewTimingWheel(time.Second, timingWheelSlots, clean)
-	lang.Must(err)
+	logx.Must(err)
 
 	proc.AddShutdownListener(func() {
 		timingWheel.Drain(clean)

core/stores/kv/store_test.go

@@ -212,10 +212,12 @@ func TestRedis_Persist(t *testing.T) {
 		assert.Nil(t, err)
 		assert.False(t, ok)
 		err = client.Expire("key", 5)
+		assert.Nil(t, err)
 		ok, err = client.Persist("key")
 		assert.Nil(t, err)
 		assert.True(t, ok)
 		err = client.Expireat("key", time.Now().Unix()+5)
+		assert.Nil(t, err)
 		ok, err = client.Persist("key")
 		assert.Nil(t, err)
 		assert.True(t, ok)
@@ -379,7 +381,7 @@ func TestRedis_SortedSet(t *testing.T) {
 		rank, err := client.Zrank("key", "value2")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(1), rank)
-		rank, err = client.Zrank("key", "value4")
+		_, err = client.Zrank("key", "value4")
 		assert.Equal(t, redis.Nil, err)
 		num, err := client.Zrem("key", "value2", "value3")
 		assert.Nil(t, err)

core/stores/redis/redis_test.go

@@ -249,10 +249,12 @@ func TestRedis_Persist(t *testing.T) {
 		assert.Nil(t, err)
 		assert.False(t, ok)
 		err = client.Expire("key", 5)
+		assert.Nil(t, err)
 		ok, err = client.Persist("key")
 		assert.Nil(t, err)
 		assert.True(t, ok)
 		err = client.Expireat("key", time.Now().Unix()+5)
+		assert.Nil(t, err)
 		ok, err = client.Persist("key")
 		assert.Nil(t, err)
 		assert.True(t, ok)
@@ -447,7 +449,7 @@ func TestRedis_SortedSet(t *testing.T) {
 		rank, err := client.Zrank("key", "value2")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(1), rank)
-		rank, err = client.Zrank("key", "value4")
+		_, err = client.Zrank("key", "value4")
 		assert.Equal(t, Nil, err)
 		num, err := client.Zrem("key", "value2", "value3")
 		assert.Nil(t, err)
@@ -558,6 +560,7 @@ func TestRedis_Pipelined(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Equal(t, "1", value)
 		score, err := client.Zscore("zadd", "zadd")
+		assert.Nil(t, err)
 		assert.Equal(t, int64(12), score)
 	})
 }

core/stringx/replacer.go

@@ -63,11 +63,6 @@ func (r *replacer) Replace(text string) string {
 			i = j - 1
 			builder.WriteString(r.mapping[string(chars[start:end])])
 		} else {
-			if j < size {
-				end = j + 1
-			} else {
-				end = size
-			}
 			builder.WriteRune(chars[i])
 		}
 		start = -1

core/stringx/trie.go

@@ -2,7 +2,11 @@ package stringx
 
 import "github.com/tal-tech/go-zero/core/lang"
 
+const defaultMask = '*'
+
 type (
+	TrieOption func(trie *trieNode)
+
 	Trie interface {
 		Filter(text string) (string, []string, bool)
 		FindKeywords(text string) []string
@@ -10,6 +14,7 @@ type (
 
 	trieNode struct {
 		node
+		mask rune
 	}
 
 	scope struct {
@@ -18,8 +23,15 @@ type (
 	}
 )
 
-func NewTrie(words []string) Trie {
+func NewTrie(words []string, opts ...TrieOption) Trie {
 	n := new(trieNode)
+
+	for _, opt := range opts {
+		opt(n)
+	}
+	if n.mask == 0 {
+		n.mask = defaultMask
+	}
 	for _, word := range words {
 		n.add(word)
 	}
@@ -114,6 +126,12 @@ func (n *trieNode) findKeywordScopes(chars []rune) []scope {
 
 func (n *trieNode) replaceWithAsterisk(chars []rune, start, stop int) {
 	for i := start; i < stop; i++ {
-		chars[i] = '*'
+		chars[i] = n.mask
+	}
+}
+
+func WithMask(mask rune) TrieOption {
+	return func(n *trieNode) {
+		n.mask = mask
 	}
 }

core/stringx/trie_test.go

@@ -109,25 +109,25 @@ func TestTrie(t *testing.T) {
 func TestTrieSingleWord(t *testing.T) {
 	trie := NewTrie([]string{
 		"闹",
-	})
+	}, WithMask('#'))
 	output, keywords, ok := trie.Filter("今晚真热闹")
 	assert.ElementsMatch(t, []string{"闹"}, keywords)
 	assert.True(t, ok)
-	assert.Equal(t, "今晚真热*", output)
+	assert.Equal(t, "今晚真热#", output)
 }
 
 func TestTrieOverlap(t *testing.T) {
 	trie := NewTrie([]string{
 		"一二三四五",
 		"二三四五六七八",
-	})
+	}, WithMask('#'))
 	output, keywords, ok := trie.Filter("零一二三四五六七八九十")
 	assert.ElementsMatch(t, []string{
 		"一二三四五",
 		"二三四五六七八",
 	}, keywords)
 	assert.True(t, ok)
-	assert.Equal(t, "零********九十", output)
+	assert.Equal(t, "零########九十", output)
 }
 
 func TestTrieNested(t *testing.T) {
@@ -135,7 +135,7 @@ func TestTrieNested(t *testing.T) {
 		"一二三",
 		"一二三四五",
 		"一二三四五六七八",
-	})
+	}, WithMask('#'))
 	output, keywords, ok := trie.Filter("零一二三四五六七八九十")
 	assert.ElementsMatch(t, []string{
 		"一二三",
@@ -143,7 +143,7 @@ func TestTrieNested(t *testing.T) {
 		"一二三四五六七八",
 	}, keywords)
 	assert.True(t, ok)
-	assert.Equal(t, "零********九十", output)
+	assert.Equal(t, "零########九十", output)
 }
 
 func BenchmarkTrie(b *testing.B) {

core/sysx/host.go

@@ -3,7 +3,7 @@ package sysx
 import (
 	"os"
 
-	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/stringx"
 )
 
 var hostname string
@@ -11,7 +11,9 @@ var hostname string
 func init() {
 	var err error
 	hostname, err = os.Hostname()
-	lang.Must(err)
+	if err != nil {
+		hostname = stringx.RandId()
+	}
 }
 
 func Hostname() string {

core/utils/report.go

@@ -1,5 +0,0 @@
-package utils
-
-func Report(content string) {
-	// TODO: implement the report method
-}

doc/goctl.md

@@ -3,8 +3,8 @@
 ## goctl用途
 * 定义api请求
 * 根据定义的api自动生成golang(后端), java(iOS & Android), typescript(web & 晓程序)，dart(flutter)
-* 生成MySQL CURD (https://goctl.xiaoheiban.cn)
-*  生成MongoDB CURD (https://goctl.xiaoheiban.cn)
+* 生成MySQL CURD+Cache
+*  生成MongoDB CURD+Cache
 
 ## goctl使用说明
 #### goctl参数说明
@@ -179,23 +179,38 @@ service user-api {
   * 在定义的get/post/put/delete等请求的handler和logic里增加处理业务逻辑的代码
 
 #### 根据定义好的api文件生成java代码
-	`goctl api java -api user/user.api -dir ./src`
+```shell
+goctl api java -api user/user.api -dir ./src
+```
 
 #### 根据定义好的api文件生成typescript代码
-	`goctl api ts -api user/user.api -dir ./src -webapi ***`
-	
-	ts需要指定webapi所在目录
+```shell
+goctl api ts -api user/user.api -dir ./src -webapi ***
+
+ts需要指定webapi所在目录
+```
 
 #### 根据定义好的api文件生成Dart代码
-	`goctl api dart -api user/user.api -dir ./src`
+```shell
+goctl api dart -api user/user.api -dir ./src
+```
+
+## 根据mysql ddl或者datasource生成model文件
+
+```shell script
+$ goctl model mysql -src={filename} -dir={dir} -cache={true|false}
+```
+详情参考[model文档](https://github.com/tal-tech/go-zero/blob/master/tools/goctl/model/sql/README.MD)
 
 ## 根据定义好的简单go文件生成mongo代码文件(仅限golang使用)  
-    `goctl model mongo -src {{yourDir}}/xiao/service/xhb/user/model/usermodel.go -cache yes`
-    
-    -src需要提供简单的usermodel.go文件，里面只需要提供一个结构体即可
-    -cache 控制是否需要缓存 yes=需要 no=不需要
-    src 示例代码如下
-  ```
+```shell
+goctl model mongo -src {{yourDir}}/xiao/service/xhb/user/model/usermodel.go -cache yes
+
+-src需要提供简单的usermodel.go文件，里面只需要提供一个结构体即可
+-cache 控制是否需要缓存 yes=需要 no=不需要
+src 示例代码如下
+```
+  ```go
 package model
     
 type User struct {
@@ -210,7 +225,7 @@ type User struct {
      o是改字段需要生产的操作函数 可以取得get,find,set 分别表示生成返回单个对象的查询方法，返回多个对象的查询方法，设置该字段方法  
      生成的目标文件会覆盖该简单go文件  
 
-## goctl rpc生成
+## goctl rpc生成（业务剥离中，暂未开放）
 
   命令 `goctl rpc proto -proto ${proto} -service ${serviceName} -project ${projectName} -dir ${directory} -shared ${shared}`  
   如： `goctl rpc proto -proto test.proto  -service test -project xjy  -dir .`  
@@ -261,5 +276,4 @@ type User struct {
     │   └── test.go [强制覆盖更新]
     └── test.proto
   ```
-  - 注意 ：目前rpc目录生成的proto文件暂不支持import外部proto文件   
-* 如有不理解的地方，随时问Kim/Kevin
+  - 注意 ：目前rpc目录生成的proto文件暂不支持import外部proto文件   

doc/keywords.md

@@ -0,0 +1,86 @@
+# 高效的关键词替换和敏感词过滤工具
+
+## 1. 算法介绍
+
+利用高效的Trie树建立关键词树，如下图所示，然后依次查找字符串中的相连字符是否形成树的一条路径
+
+<img src="images/trie.png" alt="trie" width="350" />
+
+发现掘金上[这篇文章](https://juejin.im/post/6844903750490914829)写的比较详细，可以一读，具体原理在此不详述。
+
+## 2. 关键词替换
+
+支持关键词重叠，自动选用最长的关键词，代码示例如下：
+
+```go
+replacer := stringx.NewReplacer(map[string]string{
+  "日本":    "法国",
+  "日本的首都": "东京",
+  "东京":    "日本的首都",
+})
+fmt.Println(replacer.Replace("日本的首都是东京"))
+```
+
+可以得到：
+```
+东京是日本的首都
+```
+
+示例代码见`example/stringx/replace/replace.go`
+
+## 3. 查找敏感词
+
+代码示例如下：
+
+```go
+filter := stringx.NewTrie([]string{
+  "AV演员",
+  "苍井空",
+  "AV",
+  "日本AV女优",
+  "AV演员色情",
+})
+keywords := filter.FindKeywords("日本AV演员兼电视、电影演员。苍井空AV女优是xx出道, 日本AV女优们最精彩的表演是AV演员色情表演")
+fmt.Println(keywords)
+```
+
+可以得到：
+
+```
+[苍井空 日本AV女优 AV演员色情 AV AV演员]
+```
+
+## 4. 敏感词过滤
+
+代码示例如下：
+
+```go
+filter := stringx.NewTrie([]string{
+  "AV演员",
+  "苍井空",
+  "AV",
+  "日本AV女优",
+  "AV演员色情",
+}, stringx.WithMask('?')) // 默认替换为*
+safe, keywords, found := filter.Filter("日本AV演员兼电视、电影演员。苍井空AV女优是xx出道, 日本AV女优们最精彩的表演是AV演员色情表演")
+fmt.Println(safe)
+fmt.Println(keywords)
+fmt.Println(found)
+```
+
+可以得到：
+
+```
+日本????兼电视、电影演员。?????女优是xx出道, ??????们最精彩的表演是??????表演
+[苍井空 日本AV女优 AV演员色情 AV AV演员]
+true
+```
+
+示例代码见`example/stringx/filter/filter.go`
+
+## 5. Benchmark
+
+| Sentences | Keywords | Regex    | go-zero |
+| --------- | -------- | -------- | ------- |
+| 10000     | 10000    | 16min10s | 27.2ms  |
+

doc/shorturl.md

@@ -0,0 +1,507 @@
+# 快速构建高并发微服务
+
+## 0. 为什么说做好微服务很难？
+
+要想做好微服务，我们需要理解和掌握的知识点非常多，从几个维度上来说：
+
+* 基本功能层面
+  1. 并发控制&限流，避免服务被突发流量击垮
+  2. 服务注册与服务发现，确保能够动态侦测增减的节点
+  3. 负载均衡，需要根据节点承受能力分发流量
+  4. 超时控制，避免对已超时请求做无用功
+  5. 熔断设计，快速失败，保障故障节点的恢复能力
+
+* 高阶功能层面
+  1. 请求认证，确保每个用户只能访问自己的数据
+  2. 链路追踪，用于理解整个系统和快速定位特定请求的问题
+  3. 日志，用于数据收集和问题定位
+  4. 可观测性，没有度量就没有优化
+
+对于其中每一点，我们都需要用很长的篇幅来讲述其原理和实现，那么对我们后端开发者来说，要想把这些知识点都掌握并落实到业务系统里，难度是非常大的，不过我们可以依赖已经被大流量验证过的框架体系。[go-zero微服务框架](https://github.com/tal-tech/go-zero)就是为此而生。
+
+另外，我们始终秉承**工具大于约定和文档**的理念。我们希望尽可能减少开发人员的心智负担，把精力都投入到产生业务价值的代码上，减少重复代码的编写，所以我们开发了`goctl`工具。
+
+下面我通过短链微服务来演示通过[go-zero](https://github.com/tal-tech/go-zero)快速的创建微服务的流程，走完一遍，你就会发现：原来编写微服务如此简单！
+
+## 1. 什么是短链服务？
+
+短链服务就是将长的URL网址，通过程序计算等方式，转换为简短的网址字符串。
+
+写此短链服务是为了从整体上演示go-zero构建完整微服务的过程，算法和实现细节尽可能简化了，所以这不是一个高阶的短链服务。
+
+## 2. 短链微服务架构图
+
+<img src="images/shorturl-arch.png" alt="架构图" width="800" />
+
+* 这里只用了`Transform RPC`一个微服务，并不是说API Gateway只能调用一个微服务，只是为了最简演示API Gateway如何调用RPC微服务而已
+* 在真正项目里要尽可能每个微服务使用自己的数据库，数据边界要清晰
+
+## 3. 准备工作
+
+* 安装etcd, mysql, redis
+
+* 安装goctl工具
+
+  ```shell
+  export GO111MODULE=on export GOPROXY=https://goproxy.cn/,direct go get github.com/tal-tech/go-zero/tools/goctl
+  ```
+
+* 创建工作目录`shorturl`
+
+* 在`shorturl`目录下执行`go mod init shorturl`初始化`go.mod`
+
+## 4. 编写API Gateway代码
+
+* 通过goctl生成`api/shorturl.api`并编辑，为了简洁，去除了文件开头的`info`，代码如下：
+
+  ```go
+  type (
+  	expandReq struct {
+  		shorten string `form:"shorten"`
+  	}
+  
+  	expandResp struct {
+  		url string `json:"url"`
+  	}
+  )
+  
+  type (
+  	shortenReq struct {
+  		url string `form:"url"`
+  	}
+  
+  	shortenResp struct {
+  		shorten string `json:"shorten"`
+  	}
+  )
+  
+  service shorturl-api {
+  	@server(
+  		handler: ShortenHandler
+  	)
+  	get /shorten(shortenReq) returns(shortenResp)
+  
+  	@server(
+  		handler: ExpandHandler
+  	)
+  	get /expand(expandReq) returns(expandResp)
+  }
+  ```
+
+  type用法和go一致，service用来定义get/post/head/delete等api请求，解释如下：
+
+  * `service shorturl-api {`这一行定义了service名字
+  * `@server`部分用来定义server端用到的属性
+  * `handler`定义了服务端handler名字
+  * `get /shorten(shortenReq) returns(shortenResp)`定义了get方法的路由、请求参数、返回参数等
+
+* 使用goctl生成API Gateway代码
+
+  ```shell
+  goctl api go -api shorturl.api -dir .
+  ```
+
+  生成的文件结构如下：
+
+  ```
+  .
+  ├── api
+  │   ├── etc
+  │   │   └── shorturl-api.yaml         // 配置文件
+  │   ├── internal
+  │   │   ├── config
+  │   │   │   └── config.go             // 定义配置
+  │   │   ├── handler
+  │   │   │   ├── expandhandler.go      // 实现expandHandler
+  │   │   │   ├── routes.go             // 定义路由处理
+  │   │   │   └── shortenhandler.go     // 实现shortenHandler
+  │   │   ├── logic
+  │   │   │   ├── expandlogic.go        // 实现ExpandLogic
+  │   │   │   └── shortenlogic.go       // 实现ShortenLogic
+  │   │   ├── svc
+  │   │   │   └── servicecontext.go     // 定义ServiceContext
+  │   │   └── types
+  │   │       └── types.go              // 定义请求、返回结构体
+  │   ├── shorturl.api
+  │   └── shorturl.go                   // main入口定义
+  ├── go.mod
+  └── go.sum
+  ```
+
+* 启动API Gateway服务，默认侦听在8888端口
+
+  ```shell
+  go run shorturl.go -f etc/shorturl-api.yaml
+  ```
+
+* 测试API Gateway服务
+
+  ```shell
+  curl -i "http://localhost:8888/shorten?url=http://www.xiaoheiban.cn"
+  ```
+
+  返回如下：
+
+  ```http
+  HTTP/1.1 200 OK
+  Content-Type: application/json
+  Date: Thu, 27 Aug 2020 14:31:39 GMT
+  Content-Length: 15
+  
+  {"shortUrl":""}
+  ```
+
+  可以看到我们API Gateway其实啥也没干，就返回了个空值，接下来我们会在rpc服务里实现业务逻辑
+
+* 可以修改`internal/svc/servicecontext.go`来传递服务依赖（如果需要）
+
+* 实现逻辑可以修改`internal/logic`下的对应文件
+
+* 可以通过`goctl`生成各种客户端语言的api调用代码
+
+* 到这里，你已经可以通过goctl生成客户端代码给客户端同学并行开发了，支持多种语言，详见文档
+
+## 5. 编写transform rpc服务
+
+* 在`rpc/transform`目录下编写`transform.proto`文件
+
+  可以通过命令生成proto文件模板
+
+  ```shell
+  goctl rpc template -o transform.proto
+  ```
+
+  修改后文件内容如下：
+
+  ```protobuf
+  syntax = "proto3";
+  
+  package transform;
+  
+  message expandReq {
+      string shorten = 1;
+  }
+  
+  message expandResp {
+      string url = 1;
+  }
+  
+  message shortenReq {
+      string url = 1;
+  }
+  
+  message shortenResp {
+      string shorten = 1;
+  }
+  
+  service transformer {
+      rpc expand(expandReq) returns(expandResp);
+      rpc shorten(shortenReq) returns(shortenResp);
+  }
+  ```
+
+* 用`goctl`生成rpc代码，在`rpc/transform`目录下执行命令
+
+  ```shell
+  goctl rpc proto -src transform.proto
+  ```
+
+  文件结构如下：
+
+  ```
+  rpc/transform
+  ├── etc
+  │   └── transform.yaml              // 配置文件
+  ├── internal
+  │   ├── config
+  │   │   └── config.go               // 配置定义
+  │   ├── logic
+  │   │   ├── expandlogic.go          // expand业务逻辑在这里实现
+  │   │   └── shortenlogic.go         // shorten业务逻辑在这里实现
+  │   ├── server
+  │   │   └── transformerserver.go    // 调用入口, 不需要修改
+  │   └── svc
+  │       └── servicecontext.go       // 定义ServiceContext，传递依赖
+  ├── pb
+  │   └── transform.pb.go
+  ├── transform.go                    // rpc服务main函数
+  ├── transform.proto
+  └── transformer
+      ├── transformer.go              // 提供了外部调用方法，无需修改
+      ├── transformer_mock.go         // mock方法，测试用
+      └── types.go                    // request/response结构体定义
+  ```
+
+  直接可以运行，如下：
+
+  ```shell
+  $ go run transform.go -f etc/transform.yaml
+  Starting rpc server at 127.0.0.1:8080...
+  ```
+
+  `etc/transform.yaml`文件里可以修改侦听端口等配置
+
+## 6. 修改API Gateway代码调用transform rpc服务
+
+* 修改配置文件`shorturl-api.yaml`，增加如下内容
+
+  ```yaml
+  Transform:
+    Etcd:
+      Hosts:
+        - localhost:2379
+      Key: transform.rpc
+  ```
+
+  通过etcd自动去发现可用的transform服务
+
+* 修改`internal/config/config.go`如下，增加transform服务依赖
+
+  ```go
+  type Config struct {
+  	rest.RestConf
+  	Transform rpcx.RpcClientConf     // 手动代码
+  }
+  ```
+
+* 修改`internal/svc/servicecontext.go`，如下：
+
+  ```go
+  type ServiceContext struct {
+  	Config    config.Config
+  	Transformer rpcx.Client                               // 手动代码
+  }
+  
+  func NewServiceContext(c config.Config) *ServiceContext {
+  	return &ServiceContext{
+  		Config:    c,
+  		Transformer: rpcx.MustNewClient(c.Transform),  // 手动代码
+  	}
+  }
+  ```
+
+  通过ServiceContext在不同业务逻辑之间传递依赖
+
+* 修改`internal/logic/expandlogic.go`里的`Expand`方法，如下：
+
+  ```go
+  func (l *ExpandLogic) Expand(req types.ExpandReq) (*types.ExpandResp, error) {
+    // 手动代码开始
+    trans := transformer.NewTransformer(l.svcCtx.Transformer)
+  	resp, err := trans.Expand(l.ctx, &transformer.ExpandReq{
+  		Shorten: req.Shorten,
+  	})
+  	if err != nil {
+  		return nil, err
+  	}
+  
+  	return &types.ExpandResp{
+  		Url: resp.Url,
+  	}, nil
+    // 手动代码结束
+  }
+  ```
+
+  通过调用`transformer`的`Expand`方法实现短链恢复到url
+
+* 修改`internal/logic/shortenlogic.go`，如下：
+
+  ```go
+  func (l *ShortenLogic) Shorten(req types.ShortenReq) (*types.ShortenResp, error) {
+    // 手动代码开始
+  	trans := transformer.NewTransformer(l.svcCtx.Transformer)
+  	resp, err := trans.Shorten(l.ctx, &transformer.ShortenReq{
+  		Url: req.Url,
+  	})
+  	if err != nil {
+  		return nil, err
+  	}
+  
+  	return &types.ShortenResp{
+  		Shorten: resp.Shorten,
+  	}, nil
+    // 手动代码结束
+  }
+  ```
+
+  通过调用`transformer`的`Shorten`方法实现url到短链的变换
+
+  至此，API Gateway修改完成，虽然贴的代码多，但是期中修改的是很少的一部分，为了方便理解上下文，我贴了完整代码，接下来处理CRUD+cache
+
+## 7. 定义数据库表结构，并生成CRUD+cache代码
+
+* shorturl下创建`rpc/transform/model`目录：`mkdir -p rpc/transform/model`
+
+* 在rpc/transform/model目录下编写创建shorturl表的sql文件`shorturl.sql`，如下：
+
+  ```sql
+  CREATE TABLE `shorturl`
+  (
+    `shorten` varchar(255) NOT NULL COMMENT 'shorten key',
+    `url` varchar(255) NOT NULL COMMENT 'original url',
+    PRIMARY KEY(`shorten`)
+  ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+  ```
+
+* 创建DB和table
+
+  ```sql
+  create database gozero;
+  ```
+
+  ```sql
+  source shorturl.sql;
+  ```
+
+* 在`rpc/transform/model`目录下执行如下命令生成CRUD+cache代码，`-c`表示使用`redis cache`
+
+  ```shell
+  goctl model mysql ddl -c -src shorturl.sql -dir .
+  ```
+
+  也可以用`datasource`命令代替`ddl`来指定数据库链接直接从schema生成
+
+  生成后的文件结构如下：
+
+  ```
+  rpc/transform/model
+  ├── shorturl.sql
+  ├── shorturlmodel.go              // CRUD+cache代码
+  └── vars.go                       // 定义常量和变量
+  ```
+
+## 8. 修改shorten/expand rpc代码调用crud+cache代码
+
+* 修改`rpc/transform/etc/transform.yaml`，增加如下内容：
+
+  ```yaml
+  DataSource: root:@tcp(localhost:3306)/gozero
+  Table: shorturl
+  Cache:
+    - Host: localhost:6379
+  ```
+
+  可以使用多个redis作为cache，支持redis单点或者redis集群
+
+* 修改`rpc/transform/internal/config.go`，如下：
+
+  ```go
+  type Config struct {
+  	rpcx.RpcServerConf
+  	DataSource string             // 手动代码
+  	Table      string             // 手动代码
+  	Cache      cache.CacheConf    // 手动代码
+  }
+  ```
+
+  增加了mysql和redis cache配置
+
+* 修改`rpc/transform/internal/svc/servicecontext.go`，如下：
+
+  ```go
+  type ServiceContext struct {
+  	c     config.Config
+    Model *model.ShorturlModel   // 手动代码
+  }
+  
+  func NewServiceContext(c config.Config) *ServiceContext {
+  	return &ServiceContext{
+  		c:             c,
+  		Model: model.NewShorturlModel(sqlx.NewMysql(c.DataSource), c.Cache, c.Table), // 手动代码
+  	}
+  }
+  ```
+
+* 修改`rpc/transform/internal/logic/expandlogic.go`，如下：
+
+  ```go
+  func (l *ExpandLogic) Expand(in *expand.ExpandReq) (*expand.ExpandResp, error) {
+  	// 手动代码开始
+  	res, err := l.svcCtx.Model.FindOne(in.Shorten)
+  	if err != nil {
+  		return nil, err
+  	}
+  
+  	return &transform.ExpandResp{
+  		Url: res.Url,
+  	}, nil
+  	// 手动代码结束
+  }
+  ```
+
+* 修改`rpc/shorten/internal/logic/shortenlogic.go`，如下：
+
+  ```go
+  func (l *ShortenLogic) Shorten(in *shorten.ShortenReq) (*shorten.ShortenResp, error) {
+    // 手动代码开始，生成短链接
+  	key := hash.Md5Hex([]byte(in.Url))[:6]
+  	_, err := l.svcCtx.Model.Insert(model.Shorturl{
+  		Shorten: key,
+  		Url:     in.Url,
+  	})
+  	if err != nil {
+  		return nil, err
+  	}
+  
+  	return &transform.ShortenResp{
+  		Shorten: key,
+  	}, nil
+    // 手动代码结束
+  }
+  ```
+
+  至此代码修改完成，凡事手动修改的代码我加了标注
+
+## 9. 完整调用演示
+
+* shorten api调用
+
+  ```shell
+  curl -i "http://localhost:8888/shorten?url=http://www.xiaoheiban.cn"
+  ```
+
+  返回如下：
+
+  ```http
+  HTTP/1.1 200 OK
+  Content-Type: application/json
+  Date: Sat, 29 Aug 2020 10:49:49 GMT
+  Content-Length: 21
+  
+  {"shorten":"f35b2a"}
+  ```
+
+* expand api调用
+
+  ```shell
+  curl -i "http://localhost:8888/expand?shorten=f35b2a"
+  ```
+
+  返回如下：
+
+  ```http
+  HTTP/1.1 200 OK
+  Content-Type: application/json
+  Date: Sat, 29 Aug 2020 10:51:53 GMT
+  Content-Length: 34
+  
+  {"url":"http://www.xiaoheiban.cn"}
+  ```
+
+## 10. Benchmark
+
+因为写入依赖于mysql的写入速度，就相当于压mysql了，所以压测只测试了expand接口，相当于从mysql里读取并利用缓存，shorten.lua里随机从db里获取了100个热key来生成压测请求
+
+![Benchmark](images/shorturl-benchmark.png)
+
+可以看出在我的MacBook Pro上能达到3万+的qps。
+
+## 11. 总结
+
+我们一直强调**工具大于约定和文档**。
+
+go-zero不只是一个框架，更是一个建立在框架+工具基础上的，简化和规范了整个微服务构建的技术体系。
+
+我们在保持简单的同时也尽可能把微服务治理的复杂度封装到了框架内部，极大的降低了开发人员的心智负担，使得业务开发得以快速推进。
+
+通过go-zero+goctl生成的代码，包含了微服务治理的各种组件，包括：并发控制、自适应熔断、自适应降载、自动缓存控制等，可以轻松部署以承载巨大访问量。

example/breaker/main.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/tal-tech/go-zero/core/breaker"
 	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
 	"gopkg.in/cheggaaa/pb.v1"
 )
 
@@ -99,7 +100,7 @@ func main() {
 
 	gb := breaker.NewBreaker()
 	fp, err := os.Create("result.csv")
-	lang.Must(err)
+	logx.Must(err)
 	defer fp.Close()
 	fmt.Fprintln(fp, "seconds,state,googleCalls,netflixCalls")
 

example/etcd/demo/Makefile

@@ -1,13 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	#docker pull alpine
-	#docker pull golang:alpine
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/etcdmon:$(version) . -f example/etcd/demo/Dockerfile
-	#docker image prune --filter label=stage=gobuilder -f
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/etcdmon:$(version)
-
-deploy: push
-	kubectl -n xx-xiaoheiban set image deployment/etcdmon-deployment etcdmon=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/etcdmon:$(version)

example/etcd/demo/etcdmon.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: etcdmon
-  namespace: discov
-spec:
-  containers:
-  - name: etcdmon
-    image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/etcdmon:v200620093045
-    imagePullPolicy: Always
-    env:
-    - name: ETCD_CLUSTER
-      value: etcd.discov:2379
-  imagePullSecrets:
-  - name: aliyun

example/etcd/pub/Makefile

@@ -1,11 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/pub:$(version) . -f example/etcd/pub/Dockerfile
-	docker image prune --filter label=stage=gobuilder -f
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/pub:$(version)
-
-deploy: push
-	kubectl -n adhoc set image deployment/pub-deployment pub=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/pub:$(version)

example/etcd/pub/pub.yaml

@@ -1,26 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pub-deployment
-  namespace: adhoc
-  labels:
-    app: pub
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: pub
-  template:
-    metadata:
-      labels:
-        app: pub
-    spec:
-      containers:
-      - name: pub
-        image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/pub:v200213172101
-        command:
-        - /app/pub
-        - -v
-        - ccc
-      imagePullSecrets:
-      - name: aliyun

example/etcd/sub/Makefile

@@ -1,11 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/sub:$(version) . -f example/etcd/sub/Dockerfile
-	docker image prune --filter label=stage=gobuilder -f
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/sub:$(version)
-
-deploy: push
-	kubectl -n adhoc set image deployment/sub-deployment sub=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/sub:$(version)

example/etcd/sub/sub.go

@@ -5,12 +5,12 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/discov"
-	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
 )
 
 func main() {
 	sub, err := discov.NewSubscriber([]string{"etcd.discovery:2379"}, "028F2C35852D", discov.Exclusive())
-	lang.Must(err)
+	logx.Must(err)
 
 	ticker := time.NewTicker(time.Second * 3)
 	defer ticker.Stop()

example/etcd/sub/sub.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: sub
-  name: sub
-  namespace: adhoc
-spec:
-  containers:
-    - command:
-        - /app/sub
-      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/sub:v200213220509
-      name: sub
-  imagePullSecrets:
-    - name: aliyun
-  restartPolicy: Always

example/graceful/dns/api/Makefile

@@ -1,11 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	docker pull alpine
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/graceful:$(version) . -f example/graceful/dns/api/Dockerfile
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/graceful:$(version)
-
-deploy: push
-	kubectl -n kevin set image deployment/graceful-deployment graceful=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/graceful:$(version)

example/graceful/dns/api/graceful.yaml

@@ -1,42 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: graceful
-  namespace: kevin
-spec:
-  selector:
-    app: graceful
-  type: ClusterIP
-  ports:
-    - name: graceful-port
-      port: 3333
-      targetPort: 8888
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: graceful-deployment
-  namespace: kevin
-  labels:
-    app: graceful
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: graceful
-  template:
-    metadata:
-      labels:
-        app: graceful
-    spec:
-      containers:
-        - name: graceful
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/graceful:v191022133857
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8888
-      imagePullSecrets:
-        - name: aliyun
-

example/graceful/dns/rpc/Makefile

@@ -1,11 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	docker pull alpine
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:$(version) . -f example/graceful/dns/rpc/Dockerfile
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:$(version)
-
-deploy: push
-	kubectl -n kevin set image deployment/gracefulrpc-deployment gracefulrpc=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:$(version)

example/graceful/dns/rpc/gracefulrpc.yaml

@@ -1,46 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gracefulrpc
-  namespace: kevin
-spec:
-  selector:
-    app: gracefulrpc
-  type: ClusterIP
-  clusterIP: None
-  ports:
-    - name: gracefulrpc-port
-      port: 3456
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gracefulrpc-deployment
-  namespace: kevin
-  labels:
-    app: gracefulrpc
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: gracefulrpc
-  template:
-    metadata:
-      labels:
-        app: gracefulrpc
-    spec:
-      containers:
-        - name: gracefulrpc
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:v191022143425
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 3456
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-      imagePullSecrets:
-        - name: aliyun

example/graceful/etcd/api/Makefile

@@ -1,13 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	docker pull alpine
-	docker pull golang:alpine
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/graceful:$(version) . -f example/graceful/etcd/api/Dockerfile
-	docker image prune --filter label=stage=gobuilder -f
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/graceful:$(version)
-
-deploy: push
-	kubectl -n kevin set image deployment/graceful-deployment graceful=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/graceful:$(version)

example/graceful/etcd/api/graceful.yaml

@@ -1,42 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: graceful
-  namespace: kevin
-spec:
-  selector:
-    app: graceful
-  type: ClusterIP
-  ports:
-    - name: graceful-port
-      port: 3333
-      targetPort: 8888
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: graceful-deployment
-  namespace: kevin
-  labels:
-    app: graceful
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: graceful
-  template:
-    metadata:
-      labels:
-        app: graceful
-    spec:
-      containers:
-        - name: graceful
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/graceful:v191031145905
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8888
-      imagePullSecrets:
-        - name: aliyun
-

example/graceful/etcd/rpc/Makefile

@@ -1,13 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	docker pull alpine
-	docker pull golang:alpine
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:$(version) . -f example/graceful/etcd/rpc/Dockerfile
-	docker image prune --filter label=stage=gobuilder -f
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:$(version)
-
-deploy: push
-	kubectl -n kevin set image deployment/gracefulrpc-deployment gracefulrpc=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:$(version)

example/graceful/etcd/rpc/gracefulrpc-env.yaml

@@ -1,30 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gracefulrpc-deployment
-  namespace: kevin
-  labels:
-    app: gracefulrpc
-spec:
-  replicas: 9
-  selector:
-    matchLabels:
-      app: gracefulrpc
-  template:
-    metadata:
-      labels:
-        app: gracefulrpc
-    spec:
-      containers:
-        - name: gracefulrpc
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:v191031144304
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 3456
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-      imagePullSecrets:
-        - name: aliyun

example/graceful/etcd/rpc/gracefulrpc-headless.yaml

@@ -1,41 +0,0 @@
- apiVersion: v1
- kind: Service
- metadata:
-   name: gracefulrpc
-   namespace: kevin
- spec:
-   selector:
-     app: gracefulrpc
-   type: ClusterIP
-   clusterIP: None
-   ports:
-     - name: gracefulrpc-port
-       port: 3456
-
- ---
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gracefulrpc-deployment
-  namespace: kevin
-  labels:
-    app: gracefulrpc
-spec:
-  replicas: 9
-  selector:
-    matchLabels:
-      app: gracefulrpc
-  template:
-    metadata:
-      labels:
-        app: gracefulrpc
-    spec:
-      containers:
-        - name: gracefulrpc
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:v191031144304
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 3456
-      imagePullSecrets:
-        - name: aliyun

example/graceful/etcd/rpc/gracefulrpc.yaml

@@ -1,25 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gracefulrpc-deployment
-  namespace: kevin
-  labels:
-    app: gracefulrpc
-spec:
-  replicas: 9
-  selector:
-    matchLabels:
-      app: gracefulrpc
-  template:
-    metadata:
-      labels:
-        app: gracefulrpc
-    spec:
-      containers:
-        - name: gracefulrpc
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/gracefulrpc:v191031144304
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 3456
-      imagePullSecrets:
-        - name: aliyun

example/http/breaker/client/client.go

@@ -9,12 +9,13 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/threading"
 	"gopkg.in/cheggaaa/pb.v1"
 )
 
 var (
-	freq     = flag.Int("freq", 100, "frequence")
+	freq     = flag.Int("freq", 100, "frequency")
 	duration = flag.String("duration", "10s", "duration")
 )
 
@@ -83,8 +84,8 @@ func (m *metric) reset() counting {
 	return result
 }
 
-func runRequests(url string, frequence int, metrics *metric, done <-chan lang.PlaceholderType) {
-	ticker := time.NewTicker(time.Second / time.Duration(frequence))
+func runRequests(url string, frequency int, metrics *metric, done <-chan lang.PlaceholderType) {
+	ticker := time.NewTicker(time.Second / time.Duration(frequency))
 	defer ticker.Stop()
 
 	for {
@@ -119,14 +120,14 @@ func main() {
 	flag.Parse()
 
 	fp, err := os.Create("result.csv")
-	lang.Must(err)
+	logx.Must(err)
 	defer fp.Close()
 	fmt.Fprintln(fp, "seconds,goodOk,goodFail,goodReject,goodErrs,goodUnknowns,goodDropRatio,"+
 		"heavyOk,heavyFail,heavyReject,heavyErrs,heavyUnknowns,heavyDropRatio")
 
 	var gm, hm metric
 	dur, err := time.ParseDuration(*duration)
-	lang.Must(err)
+	logx.Must(err)
 	done := make(chan lang.PlaceholderType)
 	group := threading.NewRoutineGroup()
 	group.RunSafe(func() {

example/load/main.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/tal-tech/go-zero/core/collection"
 	"github.com/tal-tech/go-zero/core/executors"
-	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/syncx"
 	"gopkg.in/cheggaaa/pb.v1"
 )
@@ -47,7 +47,7 @@ func main() {
 	lessWriter = executors.NewLessExecutor(interval * total / 100)
 
 	fp, err := os.Create("result.csv")
-	lang.Must(err)
+	logx.Must(err)
 	defer fp.Close()
 	fmt.Fprintln(fp, "second,maxFlight,flying,agressiveAvgFlying,lazyAvgFlying,bothAvgFlying")
 

example/load/simulate/client/main.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/fx"
-	"github.com/tal-tech/go-zero/core/lang"
+	"github.com/tal-tech/go-zero/core/logx"
 )
 
 var (
@@ -27,7 +27,7 @@ func main() {
 	flag.Parse()
 
 	fp, err := os.Create("result.csv")
-	lang.Must(err)
+	logx.Must(err)
 	defer fp.Close()
 	fmt.Fprintln(fp, "seconds,total,pass,fail,drop")
 

example/load/simulate/cpu/Makefile

@@ -1,13 +0,0 @@
-version := v1
-
-build:
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/shedding:$(version) . -f example/load/simulate/cpu/Dockerfile
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/shedding:$(version)
-
-deploy: push
-	kubectl apply -f shedding.yaml
-
-clean:
-	kubectl delete -f shedding.yaml

example/load/simulate/cpu/shedding.yaml

@@ -1,17 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: shedding
-  namespace: adhoc
-spec:
-  containers:
-    - name: shedding
-      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/shedding:v1
-      imagePullPolicy: Always
-      resources:
-        requests:
-          cpu: 1000m
-        limits:
-          cpu: 1000m
-  imagePullSecrets:
-    - name: aliyun

example/periodicalexecutor/pe.go

@@ -8,11 +8,11 @@ import (
 )
 
 func main() {
-	exeutor := executors.NewBulkExecutor(func(items []interface{}) {
+	executor := executors.NewBulkExecutor(func(items []interface{}) {
 		fmt.Println(len(items))
 	}, executors.WithBulkTasks(10))
 	for {
-		exeutor.Add(1)
+		executor.Add(1)
 		time.Sleep(time.Millisecond * 90)
 	}
 }

example/rpc/client/direct/Makefile

@@ -1,10 +0,0 @@
-version := v$(shell /bin/date "+%y%m%d%H%M%S")
-
-build:
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/unarydirect:$(version) . -f example/rpc/client/direct/Dockerfile
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/unarydirect:$(version)
-
-deploy: push
-	kubectl -n adhoc set image deployment/unarydirect-deployment unarydirect=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/unarydirect:$(version)

example/rpc/client/direct/unarydirect.yaml

@@ -1,23 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: unarydirect-deployment
-  namespace: adhoc
-  labels:
-    app: unarydirect
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: unarydirect
-  template:
-    metadata:
-      labels:
-        app: unarydirect
-    spec:
-      containers:
-        - name: unarydirect
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/unarydirect:v1
-          imagePullPolicy: Always
-      imagePullSecrets:
-        - name: aliyun

example/rpc/client/lb/main.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
+	"log"
 	"time"
 
 	"github.com/tal-tech/go-zero/core/discov"
@@ -10,13 +12,31 @@ import (
 	"github.com/tal-tech/go-zero/rpcx"
 )
 
+var lb = flag.String("t", "direct", "the load balancer type")
+
 func main() {
-	cli := rpcx.MustNewClient(rpcx.RpcClientConf{
-		Etcd: discov.EtcdConf{
-			Hosts: []string{"localhost:2379"},
-			Key:   "rpcx",
-		},
-	})
+	flag.Parse()
+
+	var cli rpcx.Client
+	switch *lb {
+	case "direct":
+		cli = rpcx.MustNewClient(rpcx.RpcClientConf{
+			Endpoints: []string{
+				"localhost:3456",
+				"localhost:3457",
+			},
+		})
+	case "discov":
+		cli = rpcx.MustNewClient(rpcx.RpcClientConf{
+			Etcd: discov.EtcdConf{
+				Hosts: []string{"localhost:2379"},
+				Key:   "rpcx",
+			},
+		})
+	default:
+		log.Fatal("bad load balancing type")
+	}
+
 	greet := unary.NewGreeterClient(cli.Conn())
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()

example/rpc/proxy/proxy.go

@@ -42,7 +42,7 @@ func main() {
 		ListenOn: *listen,
 	}, func(grpcServer *grpc.Server) {
 		unary.RegisterGreeterServer(grpcServer, &GreetServer{
-			RpcProxy: rpcx.NewRpcProxy(*server),
+			RpcProxy: rpcx.NewProxy(*server),
 		})
 	})
 	proxy.Start()

example/rpc/proxy/unaryproxy.yaml

@@ -1,46 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: unaryproxy
-  namespace: kevin
-spec:
-  selector:
-    app: unaryproxy
-  ports:
-    - name: unaryproxy-port
-      port: 3456
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: unaryproxy-deployment
-  namespace: kevin
-  labels:
-    app: unaryproxy
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: unaryproxy
-  template:
-    metadata:
-      labels:
-        app: unaryproxy
-    spec:
-      containers:
-        - name: unaryproxy
-          image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/unaryproxy:v1
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 3456
-          volumeMounts:
-            - name: timezone
-              mountPath: /etc/localtime
-      imagePullSecrets:
-        - name: aliyun
-      volumes:
-        - name: timezone
-          hostPath:
-            path: /usr/share/zoneinfo/Asia/Shanghai

example/rpc/server/unary/Makefile

@@ -1,11 +0,0 @@
-version := v1
-
-build:
-	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/unaryserver:$(version) . -f example/rpc/server/unary/Dockerfile
-	docker image prune --filter label=stage=gobuilder -f
-
-push: build
-	docker push registry.cn-hangzhou.aliyuncs.com/xapp/unaryserver:$(version)
-
-deploy: push
-	kubectl -n adhoc set image deployment/unaryserver-deployment unaryserver=registry-vpc.cn-hangzhou.aliyuncs.com/xapp/unaryserver:$(version)

example/rpc/server/unary/unaryserver.yaml

@@ -1,25 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: unaryserver-deployment
-  namespace: adhoc
-  labels:
-    app: unaryserver
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: unaryserver
-  template:
-    metadata:
-      labels:
-        app: unaryserver
-    spec:
-      containers:
-      - name: unaryserver
-        image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/unaryserver:v1
-        imagePullPolicy: Always
-        ports:
-        - containerPort: 3456
-      imagePullSecrets:
-        - name: aliyun

example/stringx/filter/filter.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/tal-tech/go-zero/core/stringx"
+)
+
+func main() {
+	filter := stringx.NewTrie([]string{
+		"AV演员",
+		"苍井空",
+		"AV",
+		"日本AV女优",
+		"AV演员色情",
+	}, stringx.WithMask('?'))
+	safe, keywords, found := filter.Filter("日本AV演员兼电视、电影演员。苍井空AV女优是xx出道, 日本AV女优们最精彩的表演是AV演员色情表演")
+	fmt.Println(safe)
+	fmt.Println(keywords)
+	fmt.Println(found)
+}

example/stringx/replace/replace.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/tal-tech/go-zero/core/stringx"
+)
+
+func main() {
+	replacer := stringx.NewReplacer(map[string]string{
+		"日本":    "法国",
+		"日本的首都": "东京",
+		"东京":    "日本的首都",
+	})
+	fmt.Println(replacer.Replace("日本的首都是东京"))
+}

go.mod

@@ -8,11 +8,13 @@ require (
 	github.com/alicebob/miniredis v2.5.0+incompatible
 	github.com/dchest/siphash v1.2.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/dsymonds/gotoc v0.0.0-20160928043926-5aebcfc91819
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/frankban/quicktest v1.7.2 // indirect
 	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
 	github.com/go-redis/redis v6.15.7+incompatible
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/go-xorm/builder v0.3.4
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/mock v1.4.3
@@ -22,6 +24,7 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect
+	github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
 	github.com/justinas/alice v1.2.0
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
@@ -41,6 +44,7 @@ require (
 	github.com/stretchr/testify v1.5.1
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
 	github.com/urfave/cli v1.22.4
+	github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2
 	github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb // indirect
 	go.etcd.io/etcd v0.0.0-20200402134248-51bdeb39e698
 	go.uber.org/automaxprocs v1.3.0
@@ -52,7 +56,7 @@ require (
 	golang.org/x/tools v0.0.0-20200410132612-ae9902aceb98 // indirect
 	google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f // indirect
 	google.golang.org/grpc v1.29.1
-	google.golang.org/protobuf v1.25.0 // indirect
+	google.golang.org/protobuf v1.25.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gopkg.in/yaml.v2 v2.2.8
 	honnef.co/go/tools v0.0.1-2020.1.4 // indirect

go.sum

@@ -48,6 +48,8 @@ github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4=
 github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dsymonds/gotoc v0.0.0-20160928043926-5aebcfc91819 h1:9778zj477h/VauD8kHbOtbytW2KGQefJ/wUGE5w+mzw=
+github.com/dsymonds/gotoc v0.0.0-20160928043926-5aebcfc91819/go.mod h1:MvzMVHq8BH2Ji/o8TGDocVA70byvLrAgFTxkEnmjO4Y=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -76,6 +78,10 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-xorm/builder v0.3.4 h1:FxkeGB4Cggdw3tPwutLCpfjng2jugfkg6LDMrd/KsoY=
+github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw=
+github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:9wScpmSP5A3Bk8V3XHWUcJmYTh+ZnlHVyc+A4oZYS3Y=
+github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -134,6 +140,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtg
 github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8=
+github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -266,6 +274,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 h1:YdYsPAZ2pC6Tow/nPZOPQ96O3hm/ToAkGsPLzedXERk=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
+github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ=
+github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb h1:ZkM6LRnq40pR1Ox0hTHlnpkcOTuFIDQpZ1IN8rKKhX0=
 github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=

readme.md

@@ -1,6 +1,27 @@
-# go-zero项目介绍
+# go-zero
 
-![Go](https://github.com/tal-tech/go-zero/workflows/Go/badge.svg?branch=master)
+[![Go](https://github.com/tal-tech/go-zero/workflows/Go/badge.svg?branch=master)](https://github.com/tal-tech/go-zero/actions)
+[![codecov](https://codecov.io/gh/tal-tech/go-zero/branch/master/graph/badge.svg)](https://codecov.io/gh/tal-tech/go-zero)
+[![Go Report Card](https://goreportcard.com/badge/github.com/tal-tech/go-zero)](https://goreportcard.com/report/github.com/tal-tech/go-zero)
+[![Release](https://img.shields.io/github/v/release/tal-tech/go-zero.svg?style=flat-square)](https://github.com/tal-tech/go-zero)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## 0. go-zero介绍
+
+go-zero是一个集成了各种工程实践的web和rpc框架。通过弹性设计保障了大并发服务端的稳定性，经受了充分的实战检验。
+
+go-zero 包含极简的 API 定义和生成工具 goctl，可以根据定义的 api 文件一键生成 Go, iOS, Android, Kotlin, Dart, TypeScript, JavaScript 代码，并可直接运行。
+
+使用go-zero的好处：
+
+* 轻松获得支撑千万日活服务的稳定性
+* 内建级联超时控制、限流、自适应熔断、自适应降载等微服务治理能力，无需配置和额外代码
+* 微服务治理中间件可无缝集成到其它现有框架使用
+* 极简的API描述，一键生成各端代码
+* 自动校验客户端请求参数合法性
+* 大量微服务治理和并发工具包
+
+<img src="doc/images/architecture.png" alt="架构图" width="1500" />
 
 ## 1. go-zero框架背景
 
@@ -53,47 +74,38 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架，有如下
 
 ![弹性设计](doc/images/resilience.jpg)
 
-## 4. go-zero框架收益
-
-* 保障大并发服务端的稳定性，经受了充分的实战检验
-* 极简的API定义
-* 一键生成Go, iOS, Android, Dart, TypeScript, JavaScript代码，并可直接运行
-* 服务端自动校验参数合法性
-
-## 5. go-zero近期开发计划
+## 4. go-zero近期开发计划
 
 * 自动生成API mock server，便于客户端开发
 * 自动生成服务端功能测试
 
-## 6. Installation
+## 5. Installation
 
-1. 在项目目录下通过如下命令安装：
+在项目目录下通过如下命令安装：
+
+```shell
+go get -u github.com/tal-tech/go-zero
+```
+
+## 6. Quick Start
+
+0. 完整示例请查看
+
+   [快速构建高并发微服务](doc/shorturl.md)
+
+1. 安装goctl工具
 
    ```shell
-   go get -u github.com/tal-tech/go-zero
+   export GO111MODULE=on export GOPROXY=https://goproxy.cn/,direct go get github.com/tal-tech/go-zero/tools/goctl
    ```
 
-2. 代码里导入go-zero
-
-   ```go
-   import "github.com/tal-tech/go-zero"
-   ```
-
-## 7. Quick Start
-
-1. 编译goctl工具
-
-   ```shell
-   go build tools/goctl/goctl.go
-   ```
-
-   把goctl放到$PATH的目录下，确保goctl可执行
+   确保goctl可执行
 
 2. 定义API文件，比如greet.api，可以在vs code里安装`goctl`插件，支持api语法
 
    ```go
    type Request struct {
-     Name string `path:"name"`
+     Name string `path:"name,options=you|me"` // 框架自动验证请求参数是否合法
    }
    
    type Response struct {
@@ -123,10 +135,9 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架，有如下
    生成的文件结构如下：
 
    ```
-   .
    ├── greet
    │   ├── etc
-   │   │   └── greet-api.json        // 配置文件
+   │   │   └── greet-api.yaml        // 配置文件
    │   ├── greet.go                  // main文件
    │   └── internal
    │       ├── config
@@ -141,26 +152,30 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架，有如下
    │       └── types
    │           └── types.go          // 请求、返回等类型定义
    └── greet.api                     // api描述文件
-   
-   8 directories, 9 files
    ```
    生成的代码可以直接运行：
-   
+
 ```shell
    cd greet
-   go run greet.go -f etc/greet-api.json
-   ```
-   
+   go run greet.go -f etc/greet-api.yaml
+```
+
 默认侦听在8888端口（可以在配置文件里修改），可以通过curl请求：
-   
+
 ```shell
-   ➜  go-zero git:(master) curl -w "\ncode: %{http_code}\n" http://localhost:8888/greet/from/kevin
-   {"code":0}
-   code: 200
-   ```
-   
+curl -i http://localhost:8888/greet/from/you
+```
+
+返回如下：
+
+```http
+HTTP/1.1 200 OK
+Date: Sun, 30 Aug 2020 15:32:35 GMT
+Content-Length: 0
+```
+
 编写业务代码：
-   
+
 * 可以在servicecontext.go里面传递依赖给logic，比如mysql, redis等
    * 在api定义的get/post/put/delete等请求对应的logic里增加业务处理逻辑
    
@@ -172,6 +187,18 @@ go-zero是一个集成了各种工程实践的包含web和rpc框架，有如下
    ...
    ```
 
-### 微信交流群
+## 7. Benchmark
+
+![benchmark](doc/images/benchmark.png)
+
+[测试代码见这里](https://github.com/smallnest/go-web-framework-benchmark)
+
+## 8. 文档 (逐步完善中)
+
+* [快速构建高并发微服务](doc/shorturl.md)
+* [goctl使用帮助](doc/goctl.md)
+* [关键字替换和敏感词过滤工具](doc/keywords.md)
+
+## 9. 微信交流群
 
 添加我的微信：kevwan，请注明go-zero，我拉进go-zero社区群🤝

rest/engine.go

@@ -0,0 +1,214 @@
+package rest
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/justinas/alice"
+	"github.com/tal-tech/go-zero/core/codec"
+	"github.com/tal-tech/go-zero/core/load"
+	"github.com/tal-tech/go-zero/core/stat"
+	"github.com/tal-tech/go-zero/rest/handler"
+	"github.com/tal-tech/go-zero/rest/httpx"
+	"github.com/tal-tech/go-zero/rest/internal"
+	"github.com/tal-tech/go-zero/rest/router"
+)
+
+// use 1000m to represent 100%
+const topCpuUsage = 1000
+
+var ErrSignatureConfig = errors.New("bad config for Signature")
+
+type engine struct {
+	conf                 RestConf
+	routes               []featuredRoutes
+	unauthorizedCallback handler.UnauthorizedCallback
+	unsignedCallback     handler.UnsignedCallback
+	middlewares          []Middleware
+	shedder              load.Shedder
+	priorityShedder      load.Shedder
+}
+
+func newEngine(c RestConf) *engine {
+	srv := &engine{
+		conf: c,
+	}
+	if c.CpuThreshold > 0 {
+		srv.shedder = load.NewAdaptiveShedder(load.WithCpuThreshold(c.CpuThreshold))
+		srv.priorityShedder = load.NewAdaptiveShedder(load.WithCpuThreshold(
+			(c.CpuThreshold + topCpuUsage) >> 1))
+	}
+
+	return srv
+}
+
+func (s *engine) AddRoutes(r featuredRoutes) {
+	s.routes = append(s.routes, r)
+}
+
+func (s *engine) SetUnauthorizedCallback(callback handler.UnauthorizedCallback) {
+	s.unauthorizedCallback = callback
+}
+
+func (s *engine) SetUnsignedCallback(callback handler.UnsignedCallback) {
+	s.unsignedCallback = callback
+}
+
+func (s *engine) Start() error {
+	return s.StartWithRouter(router.NewPatRouter())
+}
+
+func (s *engine) StartWithRouter(router httpx.Router) error {
+	if err := s.bindRoutes(router); err != nil {
+		return err
+	}
+
+	return internal.StartHttp(s.conf.Host, s.conf.Port, router)
+}
+
+func (s *engine) appendAuthHandler(fr featuredRoutes, chain alice.Chain,
+	verifier func(alice.Chain) alice.Chain) alice.Chain {
+	if fr.jwt.enabled {
+		if len(fr.jwt.prevSecret) == 0 {
+			chain = chain.Append(handler.Authorize(fr.jwt.secret,
+				handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
+		} else {
+			chain = chain.Append(handler.Authorize(fr.jwt.secret,
+				handler.WithPrevSecret(fr.jwt.prevSecret),
+				handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
+		}
+	}
+
+	return verifier(chain)
+}
+
+func (s *engine) bindFeaturedRoutes(router httpx.Router, fr featuredRoutes, metrics *stat.Metrics) error {
+	verifier, err := s.signatureVerifier(fr.signature)
+	if err != nil {
+		return err
+	}
+
+	for _, route := range fr.routes {
+		if err := s.bindRoute(fr, router, metrics, route, verifier); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *engine) bindRoute(fr featuredRoutes, router httpx.Router, metrics *stat.Metrics,
+	route Route, verifier func(chain alice.Chain) alice.Chain) error {
+	chain := alice.New(
+		handler.TracingHandler,
+		s.getLogHandler(),
+		handler.MaxConns(s.conf.MaxConns),
+		handler.BreakerHandler(route.Method, route.Path, metrics),
+		handler.SheddingHandler(s.getShedder(fr.priority), metrics),
+		handler.TimeoutHandler(time.Duration(s.conf.Timeout)*time.Millisecond),
+		handler.RecoverHandler,
+		handler.MetricHandler(metrics),
+		handler.PromMetricHandler(route.Path),
+		handler.MaxBytesHandler(s.conf.MaxBytes),
+		handler.GunzipHandler,
+	)
+	chain = s.appendAuthHandler(fr, chain, verifier)
+
+	for _, middleware := range s.middlewares {
+		chain = chain.Append(convertMiddleware(middleware))
+	}
+	handle := chain.ThenFunc(route.Handler)
+
+	return router.Handle(route.Method, route.Path, handle)
+}
+
+func (s *engine) bindRoutes(router httpx.Router) error {
+	metrics := s.createMetrics()
+
+	for _, fr := range s.routes {
+		if err := s.bindFeaturedRoutes(router, fr, metrics); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *engine) createMetrics() *stat.Metrics {
+	var metrics *stat.Metrics
+
+	if len(s.conf.Name) > 0 {
+		metrics = stat.NewMetrics(s.conf.Name)
+	} else {
+		metrics = stat.NewMetrics(fmt.Sprintf("%s:%d", s.conf.Host, s.conf.Port))
+	}
+
+	return metrics
+}
+
+func (s *engine) getLogHandler() func(http.Handler) http.Handler {
+	if s.conf.Verbose {
+		return handler.DetailedLogHandler
+	} else {
+		return handler.LogHandler
+	}
+}
+
+func (s *engine) getShedder(priority bool) load.Shedder {
+	if priority && s.priorityShedder != nil {
+		return s.priorityShedder
+	}
+	return s.shedder
+}
+
+func (s *engine) signatureVerifier(signature signatureSetting) (func(chain alice.Chain) alice.Chain, error) {
+	if !signature.enabled {
+		return func(chain alice.Chain) alice.Chain {
+			return chain
+		}, nil
+	}
+
+	if len(signature.PrivateKeys) == 0 {
+		if signature.Strict {
+			return nil, ErrSignatureConfig
+		} else {
+			return func(chain alice.Chain) alice.Chain {
+				return chain
+			}, nil
+		}
+	}
+
+	decrypters := make(map[string]codec.RsaDecrypter)
+	for _, key := range signature.PrivateKeys {
+		fingerprint := key.Fingerprint
+		file := key.KeyFile
+		decrypter, err := codec.NewRsaDecrypter(file)
+		if err != nil {
+			return nil, err
+		}
+
+		decrypters[fingerprint] = decrypter
+	}
+
+	return func(chain alice.Chain) alice.Chain {
+		if s.unsignedCallback != nil {
+			return chain.Append(handler.ContentSecurityHandler(
+				decrypters, signature.Expiry, signature.Strict, s.unsignedCallback))
+		} else {
+			return chain.Append(handler.ContentSecurityHandler(
+				decrypters, signature.Expiry, signature.Strict))
+		}
+	}, nil
+}
+
+func (s *engine) use(middleware Middleware) {
+	s.middlewares = append(s.middlewares, middleware)
+}
+
+func convertMiddleware(ware Middleware) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return ware(next.ServeHTTP)
+	}
+}

rest/handler/contentsecurityhandler_test.go

@@ -217,6 +217,7 @@ func TestContentSecurityHandler(t *testing.T) {
 					signature:   test.signature,
 				}
 				req, err := buildRequest(setting)
+				assert.Nil(t, err)
 				resp := httptest.NewRecorder()
 				handler.ServeHTTP(resp, req)
 				assert.Equal(t, test.statusCode, resp.Code)
@@ -249,6 +250,7 @@ func TestContentSecurityHandler_UnsignedCallback(t *testing.T) {
 		signature: "badone",
 	}
 	req, err := buildRequest(setting)
+	assert.Nil(t, err)
 	resp := httptest.NewRecorder()
 	handler.ServeHTTP(resp, req)
 	assert.Equal(t, http.StatusOK, resp.Code)
@@ -285,6 +287,7 @@ func TestContentSecurityHandler_UnsignedCallback_WrongTime(t *testing.T) {
 		fingerprint: fingerprint,
 	}
 	req, err := buildRequest(setting)
+	assert.Nil(t, err)
 	resp := httptest.NewRecorder()
 	handler.ServeHTTP(resp, req)
 	assert.Equal(t, http.StatusOK, resp.Code)

rest/httpx/responses_test.go

@@ -1,6 +1,7 @@
 package httpx
 
 import (
+	"errors"
 	"net/http"
 	"strings"
 	"testing"
@@ -17,6 +18,24 @@ func init() {
 	logx.Disable()
 }
 
+func TestError(t *testing.T) {
+	const body = "foo"
+	w := tracedResponseWriter{
+		headers: make(map[string][]string),
+	}
+	Error(&w, errors.New(body))
+	assert.Equal(t, http.StatusBadRequest, w.code)
+	assert.Equal(t, body, strings.TrimSpace(w.builder.String()))
+}
+
+func TestOk(t *testing.T) {
+	w := tracedResponseWriter{
+		headers: make(map[string][]string),
+	}
+	Ok(&w)
+	assert.Equal(t, http.StatusOK, w.code)
+}
+
 func TestOkJson(t *testing.T) {
 	w := tracedResponseWriter{
 		headers: make(map[string][]string),

rest/ngin.go

@@ -1,170 +0,0 @@
-package rest
-
-import (
-	"log"
-	"net/http"
-
-	"github.com/tal-tech/go-zero/core/logx"
-	"github.com/tal-tech/go-zero/rest/handler"
-	"github.com/tal-tech/go-zero/rest/httpx"
-)
-
-type (
-	runOptions struct {
-		start func(*engine) error
-	}
-
-	RunOption func(*Server)
-
-	Server struct {
-		ngin *engine
-		opts runOptions
-	}
-)
-
-func MustNewServer(c RestConf, opts ...RunOption) *Server {
-	engine, err := NewServer(c, opts...)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return engine
-}
-
-func NewServer(c RestConf, opts ...RunOption) (*Server, error) {
-	if err := c.SetUp(); err != nil {
-		return nil, err
-	}
-
-	server := &Server{
-		ngin: newEngine(c),
-		opts: runOptions{
-			start: func(srv *engine) error {
-				return srv.Start()
-			},
-		},
-	}
-
-	for _, opt := range opts {
-		opt(server)
-	}
-
-	return server, nil
-}
-
-func (e *Server) AddRoutes(rs []Route, opts ...RouteOption) {
-	r := featuredRoutes{
-		routes: rs,
-	}
-	for _, opt := range opts {
-		opt(&r)
-	}
-	e.ngin.AddRoutes(r)
-}
-
-func (e *Server) AddRoute(r Route, opts ...RouteOption) {
-	e.AddRoutes([]Route{r}, opts...)
-}
-
-func (e *Server) Start() {
-	handleError(e.opts.start(e.ngin))
-}
-
-func (e *Server) Stop() {
-	logx.Close()
-}
-
-func (e *Server) Use(middleware Middleware) {
-	e.ngin.use(middleware)
-}
-
-func ToMiddleware(handler func(next http.Handler) http.Handler) Middleware {
-	return func(handle http.HandlerFunc) http.HandlerFunc {
-		return handler(handle).ServeHTTP
-	}
-}
-
-func WithJwt(secret string) RouteOption {
-	return func(r *featuredRoutes) {
-		validateSecret(secret)
-		r.jwt.enabled = true
-		r.jwt.secret = secret
-	}
-}
-
-func WithJwtTransition(secret, prevSecret string) RouteOption {
-	return func(r *featuredRoutes) {
-		// why not validate prevSecret, because prevSecret is an already used one,
-		// even it not meet our requirement, we still need to allow the transition.
-		validateSecret(secret)
-		r.jwt.enabled = true
-		r.jwt.secret = secret
-		r.jwt.prevSecret = prevSecret
-	}
-}
-
-func WithMiddleware(middleware Middleware, rs ...Route) []Route {
-	routes := make([]Route, len(rs))
-
-	for i := range rs {
-		route := rs[i]
-		routes[i] = Route{
-			Method:  route.Method,
-			Path:    route.Path,
-			Handler: middleware(route.Handler),
-		}
-	}
-
-	return routes
-}
-
-func WithPriority() RouteOption {
-	return func(r *featuredRoutes) {
-		r.priority = true
-	}
-}
-
-func WithRouter(router httpx.Router) RunOption {
-	return func(server *Server) {
-		server.opts.start = func(srv *engine) error {
-			return srv.StartWithRouter(router)
-		}
-	}
-}
-
-func WithSignature(signature SignatureConf) RouteOption {
-	return func(r *featuredRoutes) {
-		r.signature.enabled = true
-		r.signature.Strict = signature.Strict
-		r.signature.Expiry = signature.Expiry
-		r.signature.PrivateKeys = signature.PrivateKeys
-	}
-}
-
-func WithUnauthorizedCallback(callback handler.UnauthorizedCallback) RunOption {
-	return func(engine *Server) {
-		engine.ngin.SetUnauthorizedCallback(callback)
-	}
-}
-
-func WithUnsignedCallback(callback handler.UnsignedCallback) RunOption {
-	return func(engine *Server) {
-		engine.ngin.SetUnsignedCallback(callback)
-	}
-}
-
-func handleError(err error) {
-	// ErrServerClosed means the server is closed manually
-	if err == nil || err == http.ErrServerClosed {
-		return
-	}
-
-	logx.Error(err)
-	panic(err)
-}
-
-func validateSecret(secret string) {
-	if len(secret) < 8 {
-		panic("secret's length can't be less than 8")
-	}
-}

rest/server.go

@@ -1,214 +1,170 @@
 package rest
 
 import (
-	"errors"
-	"fmt"
+	"log"
 	"net/http"
-	"time"
 
-	"github.com/justinas/alice"
-	"github.com/tal-tech/go-zero/core/codec"
-	"github.com/tal-tech/go-zero/core/load"
-	"github.com/tal-tech/go-zero/core/stat"
+	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/rest/handler"
 	"github.com/tal-tech/go-zero/rest/httpx"
-	"github.com/tal-tech/go-zero/rest/internal"
-	"github.com/tal-tech/go-zero/rest/router"
 )
 
-// use 1000m to represent 100%
-const topCpuUsage = 1000
-
-var ErrSignatureConfig = errors.New("bad config for Signature")
-
-type engine struct {
-	conf                 RestConf
-	routes               []featuredRoutes
-	unauthorizedCallback handler.UnauthorizedCallback
-	unsignedCallback     handler.UnsignedCallback
-	middlewares          []Middleware
-	shedder              load.Shedder
-	priorityShedder      load.Shedder
-}
-
-func newEngine(c RestConf) *engine {
-	srv := &engine{
-		conf: c,
-	}
-	if c.CpuThreshold > 0 {
-		srv.shedder = load.NewAdaptiveShedder(load.WithCpuThreshold(c.CpuThreshold))
-		srv.priorityShedder = load.NewAdaptiveShedder(load.WithCpuThreshold(
-			(c.CpuThreshold + topCpuUsage) >> 1))
+type (
+	runOptions struct {
+		start func(*engine) error
 	}
 
-	return srv
-}
+	RunOption func(*Server)
 
-func (s *engine) AddRoutes(r featuredRoutes) {
-	s.routes = append(s.routes, r)
-}
-
-func (s *engine) SetUnauthorizedCallback(callback handler.UnauthorizedCallback) {
-	s.unauthorizedCallback = callback
-}
-
-func (s *engine) SetUnsignedCallback(callback handler.UnsignedCallback) {
-	s.unsignedCallback = callback
-}
-
-func (s *engine) Start() error {
-	return s.StartWithRouter(router.NewPatRouter())
-}
-
-func (s *engine) StartWithRouter(router httpx.Router) error {
-	if err := s.bindRoutes(router); err != nil {
-		return err
+	Server struct {
+		ngin *engine
+		opts runOptions
 	}
+)
 
-	return internal.StartHttp(s.conf.Host, s.conf.Port, router)
-}
-
-func (s *engine) appendAuthHandler(fr featuredRoutes, chain alice.Chain,
-	verifier func(alice.Chain) alice.Chain) alice.Chain {
-	if fr.jwt.enabled {
-		if len(fr.jwt.prevSecret) == 0 {
-			chain = chain.Append(handler.Authorize(fr.jwt.secret,
-				handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
-		} else {
-			chain = chain.Append(handler.Authorize(fr.jwt.secret,
-				handler.WithPrevSecret(fr.jwt.prevSecret),
-				handler.WithUnauthorizedCallback(s.unauthorizedCallback)))
-		}
-	}
-
-	return verifier(chain)
-}
-
-func (s *engine) bindFeaturedRoutes(router httpx.Router, fr featuredRoutes, metrics *stat.Metrics) error {
-	verifier, err := s.signatureVerifier(fr.signature)
+func MustNewServer(c RestConf, opts ...RunOption) *Server {
+	engine, err := NewServer(c, opts...)
 	if err != nil {
-		return err
+		log.Fatal(err)
 	}
 
-	for _, route := range fr.routes {
-		if err := s.bindRoute(fr, router, metrics, route, verifier); err != nil {
-			return err
+	return engine
+}
+
+func NewServer(c RestConf, opts ...RunOption) (*Server, error) {
+	if err := c.SetUp(); err != nil {
+		return nil, err
+	}
+
+	server := &Server{
+		ngin: newEngine(c),
+		opts: runOptions{
+			start: func(srv *engine) error {
+				return srv.Start()
+			},
+		},
+	}
+
+	for _, opt := range opts {
+		opt(server)
+	}
+
+	return server, nil
+}
+
+func (e *Server) AddRoutes(rs []Route, opts ...RouteOption) {
+	r := featuredRoutes{
+		routes: rs,
+	}
+	for _, opt := range opts {
+		opt(&r)
+	}
+	e.ngin.AddRoutes(r)
+}
+
+func (e *Server) AddRoute(r Route, opts ...RouteOption) {
+	e.AddRoutes([]Route{r}, opts...)
+}
+
+func (e *Server) Start() {
+	handleError(e.opts.start(e.ngin))
+}
+
+func (e *Server) Stop() {
+	logx.Close()
+}
+
+func (e *Server) Use(middleware Middleware) {
+	e.ngin.use(middleware)
+}
+
+func ToMiddleware(handler func(next http.Handler) http.Handler) Middleware {
+	return func(handle http.HandlerFunc) http.HandlerFunc {
+		return handler(handle).ServeHTTP
+	}
+}
+
+func WithJwt(secret string) RouteOption {
+	return func(r *featuredRoutes) {
+		validateSecret(secret)
+		r.jwt.enabled = true
+		r.jwt.secret = secret
+	}
+}
+
+func WithJwtTransition(secret, prevSecret string) RouteOption {
+	return func(r *featuredRoutes) {
+		// why not validate prevSecret, because prevSecret is an already used one,
+		// even it not meet our requirement, we still need to allow the transition.
+		validateSecret(secret)
+		r.jwt.enabled = true
+		r.jwt.secret = secret
+		r.jwt.prevSecret = prevSecret
+	}
+}
+
+func WithMiddleware(middleware Middleware, rs ...Route) []Route {
+	routes := make([]Route, len(rs))
+
+	for i := range rs {
+		route := rs[i]
+		routes[i] = Route{
+			Method:  route.Method,
+			Path:    route.Path,
+			Handler: middleware(route.Handler),
 		}
 	}
 
-	return nil
+	return routes
 }
 
-func (s *engine) bindRoute(fr featuredRoutes, router httpx.Router, metrics *stat.Metrics,
-	route Route, verifier func(chain alice.Chain) alice.Chain) error {
-	chain := alice.New(
-		handler.TracingHandler,
-		s.getLogHandler(),
-		handler.MaxConns(s.conf.MaxConns),
-		handler.BreakerHandler(route.Method, route.Path, metrics),
-		handler.SheddingHandler(s.getShedder(fr.priority), metrics),
-		handler.TimeoutHandler(time.Duration(s.conf.Timeout)*time.Millisecond),
-		handler.RecoverHandler,
-		handler.MetricHandler(metrics),
-		handler.PromMetricHandler(route.Path),
-		handler.MaxBytesHandler(s.conf.MaxBytes),
-		handler.GunzipHandler,
-	)
-	chain = s.appendAuthHandler(fr, chain, verifier)
-
-	for _, middleware := range s.middlewares {
-		chain = chain.Append(convertMiddleware(middleware))
+func WithPriority() RouteOption {
+	return func(r *featuredRoutes) {
+		r.priority = true
 	}
-	handle := chain.ThenFunc(route.Handler)
-
-	return router.Handle(route.Method, route.Path, handle)
 }
 
-func (s *engine) bindRoutes(router httpx.Router) error {
-	metrics := s.createMetrics()
-
-	for _, fr := range s.routes {
-		if err := s.bindFeaturedRoutes(router, fr, metrics); err != nil {
-			return err
+func WithRouter(router httpx.Router) RunOption {
+	return func(server *Server) {
+		server.opts.start = func(srv *engine) error {
+			return srv.StartWithRouter(router)
 		}
 	}
-
-	return nil
 }
 
-func (s *engine) createMetrics() *stat.Metrics {
-	var metrics *stat.Metrics
-
-	if len(s.conf.Name) > 0 {
-		metrics = stat.NewMetrics(s.conf.Name)
-	} else {
-		metrics = stat.NewMetrics(fmt.Sprintf("%s:%d", s.conf.Host, s.conf.Port))
-	}
-
-	return metrics
-}
-
-func (s *engine) getLogHandler() func(http.Handler) http.Handler {
-	if s.conf.Verbose {
-		return handler.DetailedLogHandler
-	} else {
-		return handler.LogHandler
+func WithSignature(signature SignatureConf) RouteOption {
+	return func(r *featuredRoutes) {
+		r.signature.enabled = true
+		r.signature.Strict = signature.Strict
+		r.signature.Expiry = signature.Expiry
+		r.signature.PrivateKeys = signature.PrivateKeys
 	}
 }
 
-func (s *engine) getShedder(priority bool) load.Shedder {
-	if priority && s.priorityShedder != nil {
-		return s.priorityShedder
-	}
-	return s.shedder
-}
-
-func (s *engine) signatureVerifier(signature signatureSetting) (func(chain alice.Chain) alice.Chain, error) {
-	if !signature.enabled {
-		return func(chain alice.Chain) alice.Chain {
-			return chain
-		}, nil
-	}
-
-	if len(signature.PrivateKeys) == 0 {
-		if signature.Strict {
-			return nil, ErrSignatureConfig
-		} else {
-			return func(chain alice.Chain) alice.Chain {
-				return chain
-			}, nil
-		}
-	}
-
-	decrypters := make(map[string]codec.RsaDecrypter)
-	for _, key := range signature.PrivateKeys {
-		fingerprint := key.Fingerprint
-		file := key.KeyFile
-		decrypter, err := codec.NewRsaDecrypter(file)
-		if err != nil {
-			return nil, err
-		}
-
-		decrypters[fingerprint] = decrypter
-	}
-
-	return func(chain alice.Chain) alice.Chain {
-		if s.unsignedCallback != nil {
-			return chain.Append(handler.ContentSecurityHandler(
-				decrypters, signature.Expiry, signature.Strict, s.unsignedCallback))
-		} else {
-			return chain.Append(handler.ContentSecurityHandler(
-				decrypters, signature.Expiry, signature.Strict))
-		}
-	}, nil
-}
-
-func (s *engine) use(middleware Middleware) {
-	s.middlewares = append(s.middlewares, middleware)
-}
-
-func convertMiddleware(ware Middleware) func(http.Handler) http.Handler {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(ware(next.ServeHTTP))
+func WithUnauthorizedCallback(callback handler.UnauthorizedCallback) RunOption {
+	return func(engine *Server) {
+		engine.ngin.SetUnauthorizedCallback(callback)
+	}
+}
+
+func WithUnsignedCallback(callback handler.UnsignedCallback) RunOption {
+	return func(engine *Server) {
+		engine.ngin.SetUnsignedCallback(callback)
+	}
+}
+
+func handleError(err error) {
+	// ErrServerClosed means the server is closed manually
+	if err == nil || err == http.ErrServerClosed {
+		return
+	}
+
+	logx.Error(err)
+	panic(err)
+}
+
+func validateSecret(secret string) {
+	if len(secret) < 8 {
+		panic("secret's length can't be less than 8")
 	}
 }

rpcx/client.go

@@ -49,10 +49,10 @@ func NewClient(c RpcClientConf, options ...internal.ClientOption) (Client, error
 
 	var client Client
 	var err error
-	if len(c.Server) > 0 {
-		client, err = internal.NewDirectClient(c.Server, opts...)
+	if len(c.Endpoints) > 0 {
+		client, err = internal.NewClient(internal.BuildDirectTarget(c.Endpoints), opts...)
 	} else if err = c.Etcd.Validate(); err == nil {
-		client, err = internal.NewDiscovClient(c.Etcd.Hosts, c.Etcd.Key, opts...)
+		client, err = internal.NewClient(internal.BuildDiscovTarget(c.Etcd.Hosts, c.Etcd.Key), opts...)
 	}
 	if err != nil {
 		return nil, err
@@ -64,7 +64,7 @@ func NewClient(c RpcClientConf, options ...internal.ClientOption) (Client, error
 }
 
 func NewClientNoAuth(c discov.EtcdConf) (Client, error) {
-	client, err := internal.NewDiscovClient(c.Hosts, c.Key)
+	client, err := internal.NewClient(internal.BuildDiscovTarget(c.Hosts, c.Key))
 	if err != nil {
 		return nil, err
 	}
@@ -74,6 +74,10 @@ func NewClientNoAuth(c discov.EtcdConf) (Client, error) {
 	}, nil
 }
 
+func NewClientWithTarget(target string, opts ...internal.ClientOption) (Client, error) {
+	return internal.NewClient(target, opts...)
+}
+
 func (rc *RpcClient) Conn() *grpc.ClientConn {
 	return rc.client.Conn()
 }

rpcx/client_test.go

@@ -0,0 +1,101 @@
+package rpcx
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/rpcx/internal/mock"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/test/bufconn"
+)
+
+func init() {
+	logx.Disable()
+}
+
+func dialer() func(context.Context, string) (net.Conn, error) {
+	listener := bufconn.Listen(1024 * 1024)
+	server := grpc.NewServer()
+	mock.RegisterDepositServiceServer(server, &mock.DepositServer{})
+
+	go func() {
+		if err := server.Serve(listener); err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	return func(context.Context, string) (net.Conn, error) {
+		return listener.Dial()
+	}
+}
+
+func TestDepositServer_Deposit(t *testing.T) {
+	tests := []struct {
+		name    string
+		amount  float32
+		res     *mock.DepositResponse
+		errCode codes.Code
+		errMsg  string
+	}{
+		{
+			"invalid request with negative amount",
+			-1.11,
+			nil,
+			codes.InvalidArgument,
+			fmt.Sprintf("cannot deposit %v", -1.11),
+		},
+		{
+			"valid request with non negative amount",
+			0.00,
+			&mock.DepositResponse{Ok: true},
+			codes.OK,
+			"",
+		},
+	}
+
+	directClient := MustNewClient(RpcClientConf{
+		Endpoints: []string{"foo"},
+		App:       "foo",
+		Token:     "bar",
+		Timeout:   1000,
+	}, WithDialOption(grpc.WithInsecure()), WithDialOption(grpc.WithContextDialer(dialer())))
+	targetClient, err := NewClientWithTarget("foo", WithDialOption(grpc.WithInsecure()),
+		WithDialOption(grpc.WithContextDialer(dialer())))
+	assert.Nil(t, err)
+	clients := []Client{
+		directClient,
+		targetClient,
+	}
+	for _, tt := range tests {
+		for _, client := range clients {
+			t.Run(tt.name, func(t *testing.T) {
+				cli := mock.NewDepositServiceClient(client.Conn())
+				request := &mock.DepositRequest{Amount: tt.amount}
+				response, err := cli.Deposit(context.Background(), request)
+				if response != nil {
+					assert.True(t, len(response.String()) > 0)
+					if response.GetOk() != tt.res.GetOk() {
+						t.Error("response: expected", tt.res.GetOk(), "received", response.GetOk())
+					}
+				}
+				if err != nil {
+					if e, ok := status.FromError(err); ok {
+						if e.Code() != tt.errCode {
+							t.Error("error code: expected", codes.InvalidArgument, "received", e.Code())
+						}
+						if e.Message() != tt.errMsg {
+							t.Error("error message: expected", tt.errMsg, "received", e.Message())
+						}
+					}
+				}
+			})
+		}
+	}
+}

rpcx/config.go

@@ -21,19 +21,19 @@ type (
 	}
 
 	RpcClientConf struct {
-		Etcd    discov.EtcdConf `json:",optional"`
-		Server  string          `json:",optional=!Etcd"`
-		App     string          `json:",optional"`
-		Token   string          `json:",optional"`
-		Timeout int64           `json:",optional"`
+		Etcd      discov.EtcdConf `json:",optional"`
+		Endpoints []string        `json:",optional=!Etcd"`
+		App       string          `json:",optional"`
+		Token     string          `json:",optional"`
+		Timeout   int64           `json:",optional"`
 	}
 )
 
-func NewDirectClientConf(server, app, token string) RpcClientConf {
+func NewDirectClientConf(endpoints []string, app, token string) RpcClientConf {
 	return RpcClientConf{
-		Server: server,
-		App:    app,
-		Token:  token,
+		Endpoints: endpoints,
+		App:       app,
+		Token:     token,
 	}
 }
 

rpcx/config_test.go

@@ -0,0 +1,42 @@
+package rpcx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/discov"
+	"github.com/tal-tech/go-zero/core/service"
+	"github.com/tal-tech/go-zero/core/stores/redis"
+)
+
+func TestRpcClientConf(t *testing.T) {
+	conf := NewDirectClientConf([]string{"localhost:1234"}, "foo", "bar")
+	assert.True(t, conf.HasCredential())
+	conf = NewEtcdClientConf([]string{"localhost:1234", "localhost:5678"}, "key", "foo", "bar")
+	assert.True(t, conf.HasCredential())
+}
+
+func TestRpcServerConf(t *testing.T) {
+	conf := RpcServerConf{
+		ServiceConf: service.ServiceConf{},
+		ListenOn:    "",
+		Etcd: discov.EtcdConf{
+			Hosts: []string{"localhost:1234"},
+			Key:   "key",
+		},
+		Auth: true,
+		Redis: redis.RedisKeyConf{
+			RedisConf: redis.RedisConf{
+				Type: redis.NodeType,
+			},
+			Key: "foo",
+		},
+		StrictControl: false,
+		Timeout:       0,
+		CpuThreshold:  0,
+	}
+	assert.True(t, conf.HasEtcd())
+	assert.NotNil(t, conf.Validate())
+	conf.Redis.Host = "localhost:5678"
+	assert.Nil(t, conf.Validate())
+}

rpcx/internal/auth/credential_test.go

@@ -0,0 +1,62 @@
+package auth
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestParseCredential(t *testing.T) {
+	tests := []struct {
+		name        string
+		withNil     bool
+		withEmptyMd bool
+		app         string
+		token       string
+	}{
+		{
+			name:    "nil",
+			withNil: true,
+		},
+		{
+			name:        "empty md",
+			withEmptyMd: true,
+		},
+		{
+			name: "empty",
+		},
+		{
+			name:  "valid",
+			app:   "foo",
+			token: "bar",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			var ctx context.Context
+			if test.withNil {
+				ctx = context.Background()
+			} else if test.withEmptyMd {
+				ctx = metadata.NewIncomingContext(context.Background(), metadata.MD{})
+			} else {
+				md := metadata.New(map[string]string{
+					"app":   test.app,
+					"token": test.token,
+				})
+				ctx = metadata.NewIncomingContext(context.Background(), md)
+			}
+			cred := ParseCredential(ctx)
+			assert.False(t, cred.RequireTransportSecurity())
+			m, err := cred.GetRequestMetadata(context.Background())
+			assert.Nil(t, err)
+			assert.Equal(t, test.app, m[appKey])
+			assert.Equal(t, test.token, m[tokenKey])
+		})
+	}
+}

rpcx/internal/balancer/p2c/p2c.go

@@ -21,7 +21,7 @@ import (
 
 const (
 	Name            = "p2c_ewma"
-	decayTime       = int64(time.Millisecond * 600)
+	decayTime       = int64(time.Second * 10) // default value from finagle
 	forcePick       = int64(time.Second)
 	initSuccess     = 1000
 	throttleSuccess = initSuccess / 2

rpcx/internal/balancer/p2c/p2c_test.go

@@ -3,7 +3,9 @@ package p2c
 import (
 	"context"
 	"fmt"
+	"runtime"
 	"strconv"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -33,19 +35,31 @@ func TestP2cPicker_Pick(t *testing.T) {
 	tests := []struct {
 		name       string
 		candidates int
+		threshold  float64
 	}{
 		{
 			name:       "single",
 			candidates: 1,
+			threshold:  0.9,
+		},
+		{
+			name:       "two",
+			candidates: 2,
+			threshold:  0.5,
 		},
 		{
 			name:       "multiple",
 			candidates: 100,
+			threshold:  0.95,
 		},
 	}
 
 	for _, test := range tests {
+		test := test
 		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			const total = 10000
 			builder := new(p2cPickerBuilder)
 			ready := make(map[resolver.Address]balancer.SubConn)
 			for i := 0; i < test.candidates; i++ {
@@ -55,7 +69,9 @@ func TestP2cPicker_Pick(t *testing.T) {
 			}
 
 			picker := builder.Build(ready)
-			for i := 0; i < 10000; i++ {
+			var wg sync.WaitGroup
+			wg.Add(total)
+			for i := 0; i < total; i++ {
 				_, done, err := picker.Pick(context.Background(), balancer.PickInfo{
 					FullMethodName: "/",
 					Ctx:            context.Background(),
@@ -64,11 +80,16 @@ func TestP2cPicker_Pick(t *testing.T) {
 				if i%100 == 0 {
 					err = status.Error(codes.DeadlineExceeded, "deadline")
 				}
-				done(balancer.DoneInfo{
-					Err: err,
-				})
+				go func() {
+					runtime.Gosched()
+					done(balancer.DoneInfo{
+						Err: err,
+					})
+					wg.Done()
+				}()
 			}
 
+			wg.Wait()
 			dist := make(map[interface{}]int)
 			conns := picker.(*p2cPicker).conns
 			for _, conn := range conns {
@@ -76,7 +97,8 @@ func TestP2cPicker_Pick(t *testing.T) {
 			}
 
 			entropy := mathx.CalcEntropy(dist)
-			assert.True(t, entropy > .95, fmt.Sprintf("entropy is %f, less than .95", entropy))
+			assert.True(t, entropy > test.threshold, fmt.Sprintf("entropy is %f, less than %f",
+				entropy, test.threshold))
 		})
 	}
 }

rpcx/internal/chainclientinterceptors_test.go

@@ -0,0 +1,123 @@
+package internal
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+)
+
+func TestWithStreamClientInterceptors(t *testing.T) {
+	opts := WithStreamClientInterceptors()
+	assert.NotNil(t, opts)
+}
+
+func TestWithUnaryClientInterceptors(t *testing.T) {
+	opts := WithUnaryClientInterceptors()
+	assert.NotNil(t, opts)
+}
+
+func TestChainStreamClientInterceptors_zero(t *testing.T) {
+	var vals []int
+	interceptors := chainStreamClientInterceptors()
+	_, err := interceptors(context.Background(), nil, new(grpc.ClientConn), "/foo",
+		func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string,
+			opts ...grpc.CallOption) (grpc.ClientStream, error) {
+			vals = append(vals, 1)
+			return nil, nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1}, vals)
+}
+
+func TestChainStreamClientInterceptors_one(t *testing.T) {
+	var vals []int
+	interceptors := chainStreamClientInterceptors(func(ctx context.Context, desc *grpc.StreamDesc,
+		cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (
+		grpc.ClientStream, error) {
+		vals = append(vals, 1)
+		return streamer(ctx, desc, cc, method, opts...)
+	})
+	_, err := interceptors(context.Background(), nil, new(grpc.ClientConn), "/foo",
+		func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string,
+			opts ...grpc.CallOption) (grpc.ClientStream, error) {
+			vals = append(vals, 2)
+			return nil, nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2}, vals)
+}
+
+func TestChainStreamClientInterceptors_more(t *testing.T) {
+	var vals []int
+	interceptors := chainStreamClientInterceptors(func(ctx context.Context, desc *grpc.StreamDesc,
+		cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (
+		grpc.ClientStream, error) {
+		vals = append(vals, 1)
+		return streamer(ctx, desc, cc, method, opts...)
+	}, func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string,
+		streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		vals = append(vals, 2)
+		return streamer(ctx, desc, cc, method, opts...)
+	})
+	_, err := interceptors(context.Background(), nil, new(grpc.ClientConn), "/foo",
+		func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string,
+			opts ...grpc.CallOption) (grpc.ClientStream, error) {
+			vals = append(vals, 3)
+			return nil, nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2, 3}, vals)
+}
+
+func TestWithUnaryClientInterceptors_zero(t *testing.T) {
+	var vals []int
+	interceptors := chainUnaryClientInterceptors()
+	err := interceptors(context.Background(), "/foo", nil, nil, new(grpc.ClientConn),
+		func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+			opts ...grpc.CallOption) error {
+			vals = append(vals, 1)
+			return nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1}, vals)
+}
+
+func TestWithUnaryClientInterceptors_one(t *testing.T) {
+	var vals []int
+	interceptors := chainUnaryClientInterceptors(func(ctx context.Context, method string, req,
+		reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		vals = append(vals, 1)
+		return invoker(ctx, method, req, reply, cc, opts...)
+	})
+	err := interceptors(context.Background(), "/foo", nil, nil, new(grpc.ClientConn),
+		func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+			opts ...grpc.CallOption) error {
+			vals = append(vals, 2)
+			return nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2}, vals)
+}
+
+func TestWithUnaryClientInterceptors_more(t *testing.T) {
+	var vals []int
+	interceptors := chainUnaryClientInterceptors(func(ctx context.Context, method string, req,
+		reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		vals = append(vals, 1)
+		return invoker(ctx, method, req, reply, cc, opts...)
+	}, func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+		invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		vals = append(vals, 2)
+		return invoker(ctx, method, req, reply, cc, opts...)
+	})
+	err := interceptors(context.Background(), "/foo", nil, nil, new(grpc.ClientConn),
+		func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+			opts ...grpc.CallOption) error {
+			vals = append(vals, 3)
+			return nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2, 3}, vals)
+}

rpcx/internal/chainserverinterceptors_test.go

@@ -0,0 +1,111 @@
+package internal
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+)
+
+func TestWithStreamServerInterceptors(t *testing.T) {
+	opts := WithStreamServerInterceptors()
+	assert.NotNil(t, opts)
+}
+
+func TestWithUnaryServerInterceptors(t *testing.T) {
+	opts := WithUnaryServerInterceptors()
+	assert.NotNil(t, opts)
+}
+
+func TestChainStreamServerInterceptors_zero(t *testing.T) {
+	var vals []int
+	interceptors := chainStreamServerInterceptors()
+	err := interceptors(nil, nil, nil, func(srv interface{}, stream grpc.ServerStream) error {
+		vals = append(vals, 1)
+		return nil
+	})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1}, vals)
+}
+
+func TestChainStreamServerInterceptors_one(t *testing.T) {
+	var vals []int
+	interceptors := chainStreamServerInterceptors(func(srv interface{}, ss grpc.ServerStream,
+		info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		vals = append(vals, 1)
+		return handler(srv, ss)
+	})
+	err := interceptors(nil, nil, nil, func(srv interface{}, stream grpc.ServerStream) error {
+		vals = append(vals, 2)
+		return nil
+	})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2}, vals)
+}
+
+func TestChainStreamServerInterceptors_more(t *testing.T) {
+	var vals []int
+	interceptors := chainStreamServerInterceptors(func(srv interface{}, ss grpc.ServerStream,
+		info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		vals = append(vals, 1)
+		return handler(srv, ss)
+	}, func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		vals = append(vals, 2)
+		return handler(srv, ss)
+	})
+	err := interceptors(nil, nil, nil, func(srv interface{}, stream grpc.ServerStream) error {
+		vals = append(vals, 3)
+		return nil
+	})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2, 3}, vals)
+}
+
+func TestChainUnaryServerInterceptors_zero(t *testing.T) {
+	var vals []int
+	interceptors := chainUnaryServerInterceptors()
+	_, err := interceptors(context.Background(), nil, nil,
+		func(ctx context.Context, req interface{}) (interface{}, error) {
+			vals = append(vals, 1)
+			return nil, nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1}, vals)
+}
+
+func TestChainUnaryServerInterceptors_one(t *testing.T) {
+	var vals []int
+	interceptors := chainUnaryServerInterceptors(func(ctx context.Context, req interface{},
+		info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+		vals = append(vals, 1)
+		return handler(ctx, req)
+	})
+	_, err := interceptors(context.Background(), nil, nil,
+		func(ctx context.Context, req interface{}) (interface{}, error) {
+			vals = append(vals, 2)
+			return nil, nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2}, vals)
+}
+
+func TestChainUnaryServerInterceptors_more(t *testing.T) {
+	var vals []int
+	interceptors := chainUnaryServerInterceptors(func(ctx context.Context, req interface{},
+		info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+		vals = append(vals, 1)
+		return handler(ctx, req)
+	}, func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo,
+		handler grpc.UnaryHandler) (resp interface{}, err error) {
+		vals = append(vals, 2)
+		return handler(ctx, req)
+	})
+	_, err := interceptors(context.Background(), nil, nil,
+		func(ctx context.Context, req interface{}) (interface{}, error) {
+			vals = append(vals, 3)
+			return nil, nil
+		})
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, []int{1, 2, 3}, vals)
+}

rpcx/internal/client.go

@@ -5,12 +5,18 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/tal-tech/go-zero/rpcx/internal/balancer/p2c"
 	"github.com/tal-tech/go-zero/rpcx/internal/clientinterceptors"
+	"github.com/tal-tech/go-zero/rpcx/internal/resolver"
 	"google.golang.org/grpc"
 )
 
 const dialTimeout = time.Second * 3
 
+func init() {
+	resolver.RegisterResolver()
+}
+
 type (
 	ClientOptions struct {
 		Timeout     time.Duration
@@ -18,8 +24,26 @@ type (
 	}
 
 	ClientOption func(options *ClientOptions)
+
+	client struct {
+		conn *grpc.ClientConn
+	}
 )
 
+func NewClient(target string, opts ...ClientOption) (*client, error) {
+	opts = append(opts, WithDialOption(grpc.WithBalancerName(p2c.Name)))
+	conn, err := dial(target, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &client{conn: conn}, nil
+}
+
+func (c *client) Conn() *grpc.ClientConn {
+	return c.conn
+}
+
 func WithDialOption(opt grpc.DialOption) ClientOption {
 	return func(options *ClientOptions) {
 		options.DialOptions = append(options.DialOptions, opt)

rpcx/internal/client_test.go

@@ -0,0 +1,30 @@
+package internal
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+)
+
+func TestWithDialOption(t *testing.T) {
+	var options ClientOptions
+	agent := grpc.WithUserAgent("chrome")
+	opt := WithDialOption(agent)
+	opt(&options)
+	assert.Contains(t, options.DialOptions, agent)
+}
+
+func TestWithTimeout(t *testing.T) {
+	var options ClientOptions
+	opt := WithTimeout(time.Second)
+	opt(&options)
+	assert.Equal(t, time.Second, options.Timeout)
+}
+
+func TestBuildDialOptions(t *testing.T) {
+	agent := grpc.WithUserAgent("chrome")
+	opts := buildDialOptions(WithDialOption(agent))
+	assert.Contains(t, opts, agent)
+}

rpcx/internal/clientinterceptors/breakerinterceptor_test.go

@@ -1,12 +1,15 @@
 package clientinterceptors
 
 import (
+	"context"
+	"errors"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/tal-tech/go-zero/core/breaker"
 	"github.com/tal-tech/go-zero/core/stat"
 	rcodes "github.com/tal-tech/go-zero/rpcx/internal/codes"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -49,3 +52,30 @@ func TestBreakerInterceptorDeadlineExceeded(t *testing.T) {
 	assert.True(t, errs[err] > 0)
 	assert.True(t, errs[breaker.ErrServiceUnavailable] > 0)
 }
+
+func TestBreakerInterceptor(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{
+			name: "nil",
+			err:  nil,
+		},
+		{
+			name: "with error",
+			err:  errors.New("mock"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cc := new(grpc.ClientConn)
+			err := BreakerInterceptor(context.Background(), "/foo", nil, nil, cc,
+				func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+					opts ...grpc.CallOption) error {
+					return test.err
+				})
+			assert.Equal(t, test.err, err)
+		})
+	}
+}

rpcx/internal/clientinterceptors/durationinterceptor_test.go

@@ -0,0 +1,37 @@
+package clientinterceptors
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+)
+
+func TestDurationInterceptor(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{
+			name: "nil",
+			err:  nil,
+		},
+		{
+			name: "with error",
+			err:  errors.New("mock"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cc := new(grpc.ClientConn)
+			err := DurationInterceptor(context.Background(), "/foo", nil, nil, cc,
+				func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+					opts ...grpc.CallOption) error {
+					return test.err
+				})
+			assert.Equal(t, test.err, err)
+		})
+	}
+}

rpcx/internal/clientinterceptors/prommetricinterceptor_test.go

@@ -0,0 +1,37 @@
+package clientinterceptors
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+)
+
+func TestPromMetricInterceptor(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{
+			name: "nil",
+			err:  nil,
+		},
+		{
+			name: "with error",
+			err:  errors.New("mock"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cc := new(grpc.ClientConn)
+			err := PromMetricInterceptor(context.Background(), "/foo", nil, nil, cc,
+				func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+					opts ...grpc.CallOption) error {
+					return test.err
+				})
+			assert.Equal(t, test.err, err)
+		})
+	}
+}

rpcx/internal/clientinterceptors/timeoutinterceptor_test.go

@@ -0,0 +1,50 @@
+package clientinterceptors
+
+import (
+	"context"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+)
+
+func TestTimeoutInterceptor(t *testing.T) {
+	timeouts := []time.Duration{0, time.Millisecond * 10}
+	for _, timeout := range timeouts {
+		t.Run(strconv.FormatInt(int64(timeout), 10), func(t *testing.T) {
+			interceptor := TimeoutInterceptor(timeout)
+			cc := new(grpc.ClientConn)
+			err := interceptor(context.Background(), "/foo", nil, nil, cc,
+				func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+					opts ...grpc.CallOption) error {
+					return nil
+				},
+			)
+			assert.Nil(t, err)
+		})
+	}
+}
+
+func TestTimeoutInterceptor_timeout(t *testing.T) {
+	const timeout = time.Millisecond * 10
+	interceptor := TimeoutInterceptor(timeout)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	var wg sync.WaitGroup
+	wg.Add(1)
+	cc := new(grpc.ClientConn)
+	err := interceptor(ctx, "/foo", nil, nil, cc,
+		func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn,
+			opts ...grpc.CallOption) error {
+			defer wg.Done()
+			tm, ok := ctx.Deadline()
+			assert.True(t, ok)
+			assert.True(t, tm.Before(time.Now().Add(timeout+time.Millisecond)))
+			return nil
+		})
+	wg.Wait()
+	assert.Nil(t, err)
+}