Compare commits

...

22 Commits

Author SHA1 Message Date
Zhang Hao
17a0908a84 add test (#95) 2020-09-22 19:15:30 +08:00
Keson
9f9c24cce9 fix bug: release empty struct limit (#96) 2020-09-22 19:13:46 +08:00
kingxt
b628bc0086 goctl support import api file (#94)
* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* update doc

* remove update

* remove no need

* remove no need

* goctl add jwt support

* goctl add jwt support

* goctl add jwt support

* goctl support import

* goctl support import

Co-authored-by: kingxt <dream4kingxt@163.com>
2020-09-22 18:32:26 +08:00
kevin
be9c48da7f add tracing logs in server side and client side 2020-09-22 17:34:39 +08:00
kevin
797a90ae7d remove unnecessary tag 2020-09-21 22:41:14 +08:00
kevin
92e60a5777 use options instead of opts in error message 2020-09-21 22:37:07 +08:00
miaogaolin
46995a4d7d 修改不能编辑代码注释 (#92)
* rename file and function name

* update comments of "code generate"
2020-09-21 18:27:35 +08:00
kingxt
5e6dcac734 feature: goctl jwt (#91)
* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* update doc

* remove update

* remove no need

* remove no need

* goctl add jwt support

* goctl add jwt support

* goctl add jwt support

* goctl add jwt support

* goctl add jwt support

* goctl add jwt support

* goctl add jwt support

Co-authored-by: kingxt <dream4kingxt@163.com>
2020-09-21 16:38:23 +08:00
dylanNew
3e7e466526 fix redis error (#88)
Co-authored-by: dylan <wangdi@xiaoheiban.cn>
2020-09-21 16:37:40 +08:00
kingxt
b6b8941a18 update doc (#90)
* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* update doc

* remove update

* remove no need

* remove no need

* update jwt doc

* update jwt doc

* update jwt doc

* update jwt doc

Co-authored-by: kingxt <dream4kingxt@163.com>
2020-09-21 16:09:02 +08:00
kingxt
878fd14739 remove no need (#87)
* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* update doc

* remove update

* remove no need

* remove no need

* add jwt doc

Co-authored-by: kingxt <dream4kingxt@163.com>
2020-09-21 14:29:12 +08:00
kevin
5e99f2b85d add trace/span in http logs 2020-09-20 22:02:45 +08:00
Howie
9c23399c33 chore: fix typos (#85)
* chore: fix typos

Signed-off-by: lihaowei <haoweili35@gmail.com>

* chore: fix 2 typos
2020-09-20 14:00:31 +08:00
kevin
86d3de4c89 use package level defined contextKey as context key 2020-09-20 12:46:35 +08:00
kevin
dc17855367 printing context key friendly 2020-09-20 12:08:30 +08:00
kevin
1606a92c6e use contextType as string type 2020-09-20 12:04:49 +08:00
mlboy
029fd3ea35 fix: golint: context.WithValue should should not use basic type as key (#83)
* fix: golint: context.WithValue should should not use basic type as key

* optimiz
2020-09-20 12:01:43 +08:00
kevin
57299a7597 rename ngin to rest in goctl 2020-09-20 09:15:19 +08:00
Changkun Ou
762af9dda2 optimize AtomicError (#82)
This commit optimize AtomicError using atomic.Value. Benchmarks:

name               old time/op  new time/op  delta
AtomicError/Load-6   305ns ±11%    12ns ± 6%  -96.18%  (p=0.000 n=10+10)
AtomicError/Set-6   314ns ±16%    14ns ± 2%  -95.61%  (p=0.000 n=10+9)
2020-09-18 22:45:01 +08:00
kevin
eccfaba614 update doc 2020-09-18 22:33:40 +08:00
kevin
974c19d6d3 update rpc example 2020-09-18 18:15:39 +08:00
Zhang Hao
0f8140031a fix rpc client examle (#81) 2020-09-18 18:07:08 +08:00
49 changed files with 573 additions and 166 deletions

View File

@@ -13,15 +13,13 @@ const (
// maps as k in the error rate table
maps = 14
setScript = `
local key = KEYS[1]
for _, offset in ipairs(ARGV) do
redis.call("setbit", key, offset, 1)
redis.call("setbit", KEYS[1], offset, 1)
end
`
testScript = `
local key = KEYS[1]
for _, offset in ipairs(ARGV) do
if tonumber(redis.call("getbit", key, offset)) == 0 then
if tonumber(redis.call("getbit", KEYS[1], offset)) == 0 then
return false
end
end

View File

@@ -1,21 +1,18 @@
package errorx
import "sync"
import "sync/atomic"
type AtomicError struct {
err error
lock sync.Mutex
err atomic.Value // error
}
func (ae *AtomicError) Set(err error) {
ae.lock.Lock()
ae.err = err
ae.lock.Unlock()
ae.err.Store(err)
}
func (ae *AtomicError) Load() error {
ae.lock.Lock()
err := ae.err
ae.lock.Unlock()
return err
if v := ae.err.Load(); v != nil {
return v.(error)
}
return nil
}

View File

@@ -2,6 +2,8 @@ package errorx
import (
"errors"
"sync"
"sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
@@ -19,3 +21,53 @@ func TestAtomicErrorNil(t *testing.T) {
var err AtomicError
assert.Nil(t, err.Load())
}
func BenchmarkAtomicError(b *testing.B) {
var aerr AtomicError
wg := sync.WaitGroup{}
b.Run("Load", func(b *testing.B) {
var done uint32
go func() {
for {
if atomic.LoadUint32(&done) != 0 {
break
}
wg.Add(1)
go func() {
aerr.Set(errDummy)
wg.Done()
}()
}
}()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = aerr.Load()
}
b.StopTimer()
atomic.StoreUint32(&done, 1)
wg.Wait()
})
b.Run("Set", func(b *testing.B) {
var done uint32
go func() {
for {
if atomic.LoadUint32(&done) != 0 {
break
}
wg.Add(1)
go func() {
_ = aerr.Load()
wg.Done()
}()
}
}()
b.ResetTimer()
for i := 0; i < b.N; i++ {
aerr.Set(errDummy)
}
b.StopTimer()
atomic.StoreUint32(&done, 1)
wg.Wait()
})
}

View File

@@ -8,55 +8,60 @@ import (
"github.com/tal-tech/go-zero/core/timex"
)
const customCallerDepth = 3
const durationCallerDepth = 3
type customLog logEntry
type durationLogger logEntry
func WithDuration(d time.Duration) Logger {
return customLog{
return &durationLogger{
Duration: timex.ReprOfDuration(d),
}
}
func (l customLog) Error(v ...interface{}) {
func (l *durationLogger) Error(v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), customCallerDepth))
l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
}
}
func (l customLog) Errorf(format string, v ...interface{}) {
func (l *durationLogger) Errorf(format string, v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), customCallerDepth))
l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
}
}
func (l customLog) Info(v ...interface{}) {
func (l *durationLogger) Info(v ...interface{}) {
if shouldLog(InfoLevel) {
l.write(infoLog, levelInfo, fmt.Sprint(v...))
}
}
func (l customLog) Infof(format string, v ...interface{}) {
func (l *durationLogger) Infof(format string, v ...interface{}) {
if shouldLog(InfoLevel) {
l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
}
}
func (l customLog) Slow(v ...interface{}) {
func (l *durationLogger) Slow(v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(slowLog, levelSlow, fmt.Sprint(v...))
}
}
func (l customLog) Slowf(format string, v ...interface{}) {
func (l *durationLogger) Slowf(format string, v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
}
}
func (l customLog) write(writer io.Writer, level, content string) {
func (l *durationLogger) WithDuration(duration time.Duration) Logger {
l.Duration = timex.ReprOfDuration(duration)
return l
}
func (l *durationLogger) write(writer io.Writer, level, content string) {
l.Timestamp = getTimestamp()
l.Level = level
l.Content = content
outputJson(writer, logEntry(l))
outputJson(writer, logEntry(*l))
}

View File

@@ -0,0 +1,52 @@
package logx
import (
"log"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestWithDurationError(t *testing.T) {
var builder strings.Builder
log.SetOutput(&builder)
WithDuration(time.Second).Error("foo")
assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
}
func TestWithDurationErrorf(t *testing.T) {
var builder strings.Builder
log.SetOutput(&builder)
WithDuration(time.Second).Errorf("foo")
assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
}
func TestWithDurationInfo(t *testing.T) {
var builder strings.Builder
log.SetOutput(&builder)
WithDuration(time.Second).Info("foo")
assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
}
func TestWithDurationInfof(t *testing.T) {
var builder strings.Builder
log.SetOutput(&builder)
WithDuration(time.Second).Infof("foo")
assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
}
func TestWithDurationSlow(t *testing.T) {
var builder strings.Builder
log.SetOutput(&builder)
WithDuration(time.Second).Slow("foo")
assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
}
func TestWithDurationSlowf(t *testing.T) {
var builder strings.Builder
log.SetOutput(&builder)
WithDuration(time.Second).WithDuration(time.Hour).Slowf("foo")
assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
}

View File

@@ -15,6 +15,7 @@ import (
"strings"
"sync"
"sync/atomic"
"time"
"github.com/tal-tech/go-zero/core/iox"
"github.com/tal-tech/go-zero/core/sysx"
@@ -96,6 +97,7 @@ type (
Infof(string, ...interface{})
Slow(...interface{})
Slowf(string, ...interface{})
WithDuration(time.Duration) Logger
}
)

View File

@@ -4,54 +4,61 @@ import (
"context"
"fmt"
"io"
"time"
"github.com/tal-tech/go-zero/core/timex"
"github.com/tal-tech/go-zero/core/trace/tracespec"
)
type tracingEntry struct {
type traceLogger struct {
logEntry
Trace string `json:"trace,omitempty"`
Span string `json:"span,omitempty"`
ctx context.Context `json:"-"`
Trace string `json:"trace,omitempty"`
Span string `json:"span,omitempty"`
ctx context.Context
}
func (l tracingEntry) Error(v ...interface{}) {
func (l *traceLogger) Error(v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), customCallerDepth))
l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
}
}
func (l tracingEntry) Errorf(format string, v ...interface{}) {
func (l *traceLogger) Errorf(format string, v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), customCallerDepth))
l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
}
}
func (l tracingEntry) Info(v ...interface{}) {
func (l *traceLogger) Info(v ...interface{}) {
if shouldLog(InfoLevel) {
l.write(infoLog, levelInfo, fmt.Sprint(v...))
}
}
func (l tracingEntry) Infof(format string, v ...interface{}) {
func (l *traceLogger) Infof(format string, v ...interface{}) {
if shouldLog(InfoLevel) {
l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
}
}
func (l tracingEntry) Slow(v ...interface{}) {
func (l *traceLogger) Slow(v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(slowLog, levelSlow, fmt.Sprint(v...))
}
}
func (l tracingEntry) Slowf(format string, v ...interface{}) {
func (l *traceLogger) Slowf(format string, v ...interface{}) {
if shouldLog(ErrorLevel) {
l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
}
}
func (l tracingEntry) write(writer io.Writer, level, content string) {
func (l *traceLogger) WithDuration(duration time.Duration) Logger {
l.Duration = timex.ReprOfDuration(duration)
return l
}
func (l *traceLogger) write(writer io.Writer, level, content string) {
l.Timestamp = getTimestamp()
l.Level = level
l.Content = content
@@ -61,7 +68,7 @@ func (l tracingEntry) write(writer io.Writer, level, content string) {
}
func WithContext(ctx context.Context) Logger {
return tracingEntry{
return &traceLogger{
ctx: ctx,
}
}

View File

@@ -19,7 +19,7 @@ var mock tracespec.Trace = new(mockTrace)
func TestTraceLog(t *testing.T) {
var buf strings.Builder
ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
WithContext(ctx).(tracingEntry).write(&buf, levelInfo, testlog)
WithContext(ctx).(*traceLogger).write(&buf, levelInfo, testlog)
assert.True(t, strings.Contains(buf.String(), mockTraceId))
assert.True(t, strings.Contains(buf.String(), mockSpanId))
}

View File

@@ -345,7 +345,7 @@ func (u *Unmarshaler) processNamedFieldWithValue(field reflect.StructField, valu
options := opts.options()
if len(options) > 0 {
if !stringx.Contains(options, mapValue.(string)) {
return fmt.Errorf(`error: value "%s" for field "%s" is not defined in opts "%v"`,
return fmt.Errorf(`error: value "%s" for field "%s" is not defined in options "%v"`,
mapValue, key, options)
}
}

View File

@@ -68,6 +68,38 @@ func TestExclusiveCallDoDupSuppress(t *testing.T) {
}
}
func TestExclusiveCallDoDiffDupSuppress(t *testing.T) {
g := NewSharedCalls()
broadcast := make(chan struct{})
var calls int32
tests := []string{"e", "a", "e", "a", "b", "c", "b", "a", "c", "d", "b", "c", "d"}
var wg sync.WaitGroup
for _, key := range tests {
wg.Add(1)
go func(k string) {
<-broadcast // get all goroutines ready
_, err := g.Do(k, func() (interface{}, error) {
atomic.AddInt32(&calls, 1)
time.Sleep(10 * time.Millisecond)
return nil, nil
})
if err != nil {
t.Errorf("Do error: %v", err)
}
wg.Done()
}(key)
}
time.Sleep(100 * time.Millisecond) // let goroutines above block
close(broadcast)
wg.Wait()
if got := atomic.LoadInt32(&calls); got != 5 { // five letters
t.Errorf("number of calls = %d; want 5", got)
}
}
func TestExclusiveCallDoExDupSuppress(t *testing.T) {
g := NewSharedCalls()
c := make(chan string)

View File

@@ -0,0 +1,12 @@
package tracespec
// TracingKey is tracing key for context
var TracingKey = contextKey("X-Trace")
// contextKey a type for context key
type contextKey string
// Printing a context will reveal a fair amount of information about it.
func (c contextKey) String() string {
return "trace/tracespec context key " + string(c)
}

View File

@@ -1,3 +0,0 @@
package tracespec
const TracingKey = "X-Trace"

View File

@@ -57,6 +57,12 @@ And now, lets walk through the complete flow of quickly create a microservice
* install etcd, mysql, redis
* install protoc-gen-go
```shell
go get -u github.com/golang/protobuf/protoc-gen-go
```
* install goctl
```shell

View File

@@ -57,6 +57,12 @@
* 安装etcd, mysql, redis
* 安装`protoc-gen-go`
```shell
go get -u github.com/golang/protobuf/protoc-gen-go
```
* 安装goctl工具
```shell

Binary file not shown.

Before

Width:  |  Height:  |  Size: 191 KiB

After

Width:  |  Height:  |  Size: 141 KiB

140
doc/jwt.md Normal file
View File

@@ -0,0 +1,140 @@
# 基于go-zero实现JWT认证
关于JWT是什么大家可以看看[官网](https://jwt.io/),一句话介绍下:是可以实现服务器无状态的鉴权认证方案,也是目前最流行的跨域认证解决方案。
要实现JWT认证我们需要分成如下两个步骤
* 客户端获取JWT token。
* 服务器对客户端带来的JWT token认证。
## 1. 客户端获取JWT Token
我们定义一个协议供客户端调用获取JWT token我们新建一个目录jwt然后在目录中执行 `goctl api -o jwt.api`将生成的jwt.api改成如下
````go
type JwtTokenRequest struct {
}
type JwtTokenResponse struct {
AccessToken string `json:"access_token"`
AccessExpire int64 `json:"access_expire"`
RefreshAfter int64 `json:"refresh_after"` // 建议客户端刷新token的绝对时间
}
type GetUserRequest struct {
UserId string `json:"userId"`
}
type GetUserResponse struct {
Name string `json:"name"`
}
service jwt-api {
@server(
handler: JwtHandler
)
post /user/token(JwtTokenRequest) returns (JwtTokenResponse)
}
@server(
jwt: JwtAuth
)
service jwt-api {
@server(
handler: GetUserHandler
)
post /user/info(GetUserRequest) returns (GetUserResponse)
}
````
在服务jwt目录中执行`goctl api go -api jwt.api -dir .`
打开jwtlogic.go文件修改 `func (l *JwtLogic) Jwt(req types.JwtTokenRequest) (*types.JwtTokenResponse, error) {` 方法如下:
```go
func (l *JwtLogic) Jwt(req types.JwtTokenRequest) (*types.JwtTokenResponse, error) {
var accessExpire = l.svcCtx.Config.JwtAuth.AccessExpire
now := time.Now().Unix()
accessToken, err := l.GenToken(now, l.svcCtx.Config.JwtAuth.AccessSecret, nil, accessExpire)
if err != nil {
return nil, err
}
return &types.JwtTokenResponse{
AccessToken: accessToken,
AccessExpire: now + accessExpire,
RefreshAfter: now + accessExpire/2,
}, nil
}
func (l *JwtLogic) GenToken(iat int64, secretKey string, payloads map[string]interface{}, seconds int64) (string, error) {
claims := make(jwt.MapClaims)
claims["exp"] = iat + seconds
claims["iat"] = iat
for k, v := range payloads {
claims[k] = v
}
token := jwt.New(jwt.SigningMethodHS256)
token.Claims = claims
return token.SignedString([]byte(secretKey))
}
```
在启动服务之前我们需要修改etc/jwt-api.yaml文件如下
```yaml
Name: jwt-api
Host: 0.0.0.0
Port: 8888
JwtAuth:
AccessSecret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxx
AccessExpire: 604800
```
启动服务器然后测试下获取到的token。
```sh
➜ curl --location --request POST '127.0.0.1:8888/user/token'
{"access_token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MDEyNjE0MjksImlhdCI6MTYwMDY1NjYyOX0.6u_hpE_4m5gcI90taJLZtvfekwUmjrbNJ-5saaDGeQc","access_expire":1601261429,"refresh_after":1600959029}
```
## 2. 服务器验证JWT token
1. 在api文件中通过`jwt: JwtAuth`标记的service表示激活了jwt认证。
2. 可以阅读rest/handler/authhandler.go文件了解服务器jwt实现。
3. 修改getuserlogic.go如下
```go
func (l *GetUserLogic) GetUser(req types.GetUserRequest) (*types.GetUserResponse, error) {
return &types.GetUserResponse{Name: "kim"}, nil
}
```
* 我们先不带JWT Authorization header请求头测试下返回http status code是401符合预期。
```sh
➜ curl -w "\nhttp: %{http_code} \n" --location --request POST '127.0.0.1:8888/user/info' \
--header 'Content-Type: application/json' \
--data-raw '{
"userId": "a"
}'
http: 401
```
* 加上Authorization header请求头测试。
```sh
➜ curl -w "\nhttp: %{http_code} \n" --location --request POST '127.0.0.1:8888/user/info' \
--header 'Authorization: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MDEyNjE0MjksImlhdCI6MTYwMDY1NjYyOX0.6u_hpE_4m5gcI90taJLZtvfekwUmjrbNJ-5saaDGeQc' \
--header 'Content-Type: application/json' \
--data-raw '{
"userId": "a"
}'
{"name":"kim"}
http: 200
```
综上所述基于go-zero的JWT认证完成在真实生产环境部署时候AccessSecret, AccessExpire, RefreshAfter根据业务场景通过配置文件配置RefreshAfter 是告诉客户端什么时候该刷新JWT token了一般都需要设置过期时间前几天。

View File

@@ -60,6 +60,12 @@ And now, lets walk through the complete flow of quickly create a microservice
* install etcd, mysql, redis
* install protoc-gen-go
```
go get -u github.com/golang/protobuf/protoc-gen-go
```
* install goctl
```shell

View File

@@ -60,6 +60,12 @@
* 安装etcd, mysql, redis
* 安装`protoc-gen-go`
```shell
go get -u github.com/golang/protobuf/protoc-gen-go
```
* 安装goctl工具
```shell

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package handler
import (

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package types
type AddReq struct {

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package handler
import (

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package types
type Response struct {

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package handler
import (

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package types
type Response struct {

View File

@@ -5,6 +5,7 @@ import (
"flag"
"fmt"
"log"
"sync"
"github.com/tal-tech/go-zero/core/discov"
"github.com/tal-tech/go-zero/example/rpc/remote/stream"
@@ -33,6 +34,7 @@ func main() {
log.Fatal(err)
}
var wg sync.WaitGroup
go func() {
for {
resp, err := stm.Recv()
@@ -41,10 +43,12 @@ func main() {
}
fmt.Println("=>", resp.Greet)
wg.Done()
}
}()
for i := 0; i < 3; i++ {
wg.Add(1)
fmt.Println("<=", name)
if err = stm.Send(&stream.StreamReq{
Name: name,
@@ -52,4 +56,6 @@ func main() {
log.Fatal(err)
}
}
wg.Wait()
}

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package handler
import (

View File

@@ -1,4 +1,4 @@
// DO NOT EDIT, generated by goctl
// Code generated by goctl. DO NOT EDIT.
package types
type ExpandReq struct {

View File

@@ -19,7 +19,7 @@ Advantages of go-zero:
* improve the stability of the services with tens of millions of daily active users
* builtin chained timeout control, concurrency control, rate limit, adaptive circuit breaker, adaptive load shedding, even no configuration needed
* builtin middlewares also can be integrated into your frameworks
* simple API syntax, one command to generate couple different languages
* simple API syntax, one command to generate couple of different languages
* auto validate the request parameters from clients
* plenty of builtin microservice management and concurrent toolkits
@@ -78,7 +78,7 @@ As below, go-zero protects the system with couple layers and mechanisms:
## 4. Future development plans of go-zero
* auto generate API mock server, make the client debugging eaisier
* auto generate API mock server, make the client debugging easier
* auto generate the simple integration test for the server side just from the .api files
## 5. Installation

View File

@@ -160,6 +160,7 @@ go get -u github.com/tal-tech/go-zero
* [防止缓存击穿之进程内共享调用](doc/sharedcalls.md)
* [基于prometheus的微服务指标监控](doc/metric.md)
* [文本序列化和反序列化](doc/mapping.md)
* [快速构建jwt鉴权认证](doc/jwt.md)
## 9. 微信交流群

View File

@@ -115,7 +115,7 @@ func logBrief(r *http.Request, code int, timer *utils.ElapsedTimer, logs *intern
buf.WriteString(fmt.Sprintf("%d - %s - %s - %s - %s",
code, r.RequestURI, httpx.GetRemoteAddr(r), r.UserAgent(), timex.ReprOfDuration(duration)))
if duration > slowThreshold {
logx.Slowf("[HTTP] %d - %s - %s - %s - slowcall(%s)",
logx.WithContext(r.Context()).Slowf("[HTTP] %d - %s - %s - %s - slowcall(%s)",
code, r.RequestURI, httpx.GetRemoteAddr(r), r.UserAgent(), timex.ReprOfDuration(duration))
}
@@ -130,9 +130,9 @@ func logBrief(r *http.Request, code int, timer *utils.ElapsedTimer, logs *intern
}
if ok {
logx.Info(buf.String())
logx.WithContext(r.Context()).Info(buf.String())
} else {
logx.Error(buf.String())
logx.WithContext(r.Context()).Error(buf.String())
}
}
@@ -143,7 +143,7 @@ func logDetails(r *http.Request, response *DetailLoggedResponseWriter, timer *ut
buf.WriteString(fmt.Sprintf("%d - %s - %s\n=> %s\n",
response.writer.code, r.RemoteAddr, timex.ReprOfDuration(duration), dumpRequest(r)))
if duration > slowThreshold {
logx.Slowf("[HTTP] %d - %s - slowcall(%s)\n=> %s\n",
logx.WithContext(r.Context()).Slowf("[HTTP] %d - %s - slowcall(%s)\n=> %s\n",
response.writer.code, r.RemoteAddr, timex.ReprOfDuration(duration), dumpRequest(r))
}
@@ -157,7 +157,7 @@ func logDetails(r *http.Request, response *DetailLoggedResponseWriter, timer *ut
buf.WriteString(fmt.Sprintf("<= %s", respBuf))
}
logx.Info(buf.String())
logx.WithContext(r.Context()).Info(buf.String())
}
func isOkResponse(code int) bool {

View File

@@ -109,6 +109,18 @@ func TestParseRequired(t *testing.T) {
assert.NotNil(t, err)
}
func TestParseOptions(t *testing.T) {
v := struct {
Position int8 `form:"pos,options=1|2"`
}{}
r, err := http.NewRequest(http.MethodGet, "http://hello.com/a?pos=4", nil)
assert.Nil(t, err)
err = Parse(r, &v)
assert.NotNil(t, err)
}
func BenchmarkParseRaw(b *testing.B) {
r, err := http.NewRequest(http.MethodGet, "http://hello.com/a?name=hello&age=18&percent=3.4", nil)
if err != nil {

View File

@@ -5,7 +5,7 @@ import (
"net/http"
)
const pathVars = "pathVars"
var pathVars = contextKey("pathVars")
func Vars(r *http.Request) map[string]string {
vars, ok := r.Context().Value(pathVars).(map[string]string)
@@ -19,3 +19,9 @@ func Vars(r *http.Request) map[string]string {
func WithPathVars(r *http.Request, params map[string]string) *http.Request {
return r.WithContext(context.WithValue(r.Context(), pathVars, params))
}
type contextKey string
func (c contextKey) String() string {
return "rest/internal/context context key" + string(c)
}

View File

@@ -10,7 +10,7 @@ import (
"github.com/tal-tech/go-zero/rest/httpx"
)
const LogContext = "request_logs"
var LogContext = contextKey("request_logs")
type LogCollector struct {
Messages []string
@@ -82,3 +82,9 @@ func formatf(r *http.Request, format string, v ...interface{}) string {
func formatWithReq(r *http.Request, v string) string {
return fmt.Sprintf("(%s - %s) %s", r.RequestURI, httpx.GetRemoteAddr(r), v)
}
type contextKey string
func (c contextKey) String() string {
return "rest/internal context key " + string(c)
}

View File

@@ -65,16 +65,16 @@ func ApiFormat(path string, printToConsole bool) error {
return m
})
info, st, service, err := parser.MatchStruct(r)
apiStruct, err := parser.MatchStruct(r)
if err != nil {
return err
}
info = strings.TrimSpace(info)
if len(service) == 0 || len(st) == 0 {
info := strings.TrimSpace(apiStruct.Info)
if len(apiStruct.Service) == 0 {
return nil
}
fs, err := format.Source([]byte(strings.TrimSpace(st)))
fs, err := format.Source([]byte(strings.TrimSpace(apiStruct.StructBody)))
if err != nil {
str := err.Error()
lineNumber := strings.Index(str, ":")
@@ -93,12 +93,24 @@ func ApiFormat(path string, printToConsole bool) error {
return err
}
result := strings.Join([]string{info, string(fs), service}, "\n\n")
var result string
if len(strings.TrimSpace(info)) > 0 {
result += strings.TrimSpace(info) + "\n\n"
}
if len(strings.TrimSpace(apiStruct.Imports)) > 0 {
result += strings.TrimSpace(apiStruct.Imports) + "\n\n"
}
if len(strings.TrimSpace(string(fs))) > 0 {
result += strings.TrimSpace(string(fs)) + "\n\n"
}
if len(strings.TrimSpace(apiStruct.Service)) > 0 {
result += strings.TrimSpace(apiStruct.Service) + "\n\n"
}
if printToConsole {
_, err := fmt.Print(result)
return err
}
result = strings.TrimSpace(result)
return ioutil.WriteFile(path, []byte(result), os.ModePerm)
}

View File

@@ -53,15 +53,13 @@ func DoGenProject(apiFile, dir string, force bool) error {
logx.Must(util.MkdirIfNotExist(dir))
logx.Must(genEtc(dir, api))
logx.Must(genConfig(dir))
logx.Must(genConfig(dir, api))
logx.Must(genMain(dir, api))
logx.Must(genServiceContext(dir, api))
logx.Must(genTypes(dir, api, force))
logx.Must(genHandlers(dir, api))
logx.Must(genRoutes(dir, api, force))
logx.Must(genLogic(dir, api))
// it does not work
format(dir)
createGoModFileIfNeed(dir)
if err := backupAndSweep(apiFile); err != nil {
@@ -102,14 +100,6 @@ func backupAndSweep(apiFile string) error {
return err
}
func format(dir string) {
cmd := exec.Command("go", "fmt", "./"+dir+"...")
_, err := cmd.CombinedOutput()
if err != nil {
fmt.Println(err.Error())
}
}
func sweep() error {
keepTime := time.Now().AddDate(0, 0, -7)
return filepath.Walk(tmpDir, func(fpath string, info os.FileInfo, err error) error {

View File

@@ -3,8 +3,10 @@ package gogen
import (
"bytes"
"fmt"
"strings"
"text/template"
"github.com/tal-tech/go-zero/tools/goctl/api/spec"
"github.com/tal-tech/go-zero/tools/goctl/api/util"
"github.com/tal-tech/go-zero/tools/goctl/vars"
)
@@ -17,11 +19,18 @@ import {{.authImport}}
type Config struct {
rest.RestConf
{{.auth}}
}
`
jwtTemplate = ` struct {
AccessSecret string
AccessExpire int64
}
`
)
func genConfig(dir string) error {
func genConfig(dir string, api *spec.ApiSpec) error {
fp, created, err := util.MaybeCreateFile(dir, configDir, configFile)
if err != nil {
return err
@@ -31,11 +40,18 @@ func genConfig(dir string) error {
}
defer fp.Close()
var authNames = getAuths(api)
var auths []string
for _, item := range authNames {
auths = append(auths, fmt.Sprintf("%s %s", item, jwtTemplate))
}
var authImportStr = fmt.Sprintf("\"%s/rest\"", vars.ProjectOpenSourceUrl)
t := template.Must(template.New("configTemplate").Parse(configTemplate))
buffer := new(bytes.Buffer)
err = t.Execute(buffer, map[string]string{
"authImport": authImportStr,
"auth": strings.Join(auths, "\n"),
})
if err != nil {
return nil

View File

@@ -17,7 +17,7 @@ import (
const (
routesFilename = "routes.go"
routesTemplate = `// DO NOT EDIT, generated by goctl
routesTemplate = `// Code generated by goctl. DO NOT EDIT.
package handler
import (
@@ -81,11 +81,11 @@ func genRoutes(dir string, api *spec.ApiSpec, force bool) error {
}
var jwt string
if g.jwtEnabled {
jwt = fmt.Sprintf(", ngin.WithJwt(serverCtx.Config.%s.AccessSecret)", g.authName)
jwt = fmt.Sprintf(", rest.WithJwt(serverCtx.Config.%s.AccessSecret)", g.authName)
}
var signature string
if g.signatureEnabled {
signature = fmt.Sprintf(", ngin.WithSignature(serverCtx.Config.%s.Signature)", g.authName)
signature = fmt.Sprintf(", rest.WithSignature(serverCtx.Config.%s.Signature)", g.authName)
}
if err := gt.Execute(&builder, map[string]string{
"routes": strings.TrimSpace(gbuilder.String()),
@@ -180,6 +180,11 @@ func getRoutes(api *spec.ApiSpec) ([]group, error) {
handler: handler,
})
}
if value, ok := apiutil.GetAnnotationValue(g.Annotations, "server", "jwt"); ok {
groupedRoutes.authName = value
groupedRoutes.jwtEnabled = true
}
routes = append(routes, groupedRoutes)
}

View File

@@ -16,7 +16,7 @@ import (
const (
typesFile = "types.go"
typesTemplate = `// DO NOT EDIT, generated by goctl
typesTemplate = `// Code generated by goctl. DO NOT EDIT.
package types{{if .containsTime}}
import (
"time"

View File

@@ -13,7 +13,7 @@ import (
)
const (
componentTemplate = `// DO NOT EDIT, generated by goctl
componentTemplate = `// Code generated by goctl. DO NOT EDIT.
package com.xhb.logic.http.packet.{{.packet}}.model;
import com.xhb.logic.http.DeProguardable;

View File

@@ -6,36 +6,61 @@ import (
"fmt"
"io"
"io/ioutil"
"path/filepath"
"strings"
"github.com/tal-tech/go-zero/tools/goctl/api/spec"
"github.com/tal-tech/go-zero/tools/goctl/util"
)
type Parser struct {
r *bufio.Reader
st string
r *bufio.Reader
typeDef string
}
func NewParser(filename string) (*Parser, error) {
apiAbsPath, err := filepath.Abs(filename)
if err != nil {
return nil, err
}
api, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
info, body, service, err := MatchStruct(string(api))
apiStruct, err := MatchStruct(string(api))
if err != nil {
return nil, err
}
for _, item := range strings.Split(apiStruct.Imports, "\n") {
ip := strings.TrimSpace(item)
if len(ip) > 0 {
item := strings.TrimPrefix(item, "import")
item = strings.TrimSpace(item)
var path = item
if !util.FileExists(item) {
path = filepath.Join(filepath.Dir(apiAbsPath), item)
}
content, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
apiStruct.StructBody += "\n" + string(content)
}
}
var buffer = new(bytes.Buffer)
buffer.WriteString(info)
buffer.WriteString(service)
buffer.WriteString(apiStruct.Service)
return &Parser{
r: bufio.NewReader(buffer),
st: body,
r: bufio.NewReader(buffer),
typeDef: apiStruct.StructBody,
}, nil
}
func (p *Parser) Parse() (api *spec.ApiSpec, err error) {
api = new(spec.ApiSpec)
types, err := parseStructAst(p.st)
types, err := parseStructAst(p.typeDef)
if err != nil {
return nil, err
}

View File

@@ -3,19 +3,19 @@ package parser
import (
"bufio"
"errors"
"regexp"
"strings"
"github.com/tal-tech/go-zero/tools/goctl/api/spec"
)
// struct match
const typeRegex = `(?m)(?m)(^ *type\s+[a-zA-Z][a-zA-Z0-9_-]+\s+(((struct)\s*?\{[\w\W]*?[^\{]\})|([a-zA-Z][a-zA-Z0-9_-]+)))|(^ *type\s*?\([\w\W]+\}\s*\))`
var emptyType spec.Type
var (
emptyStrcut = errors.New("struct body not found")
emptyType spec.Type
)
type ApiStruct struct {
Info string
StructBody string
Service string
Imports string
}
func GetType(api *spec.ApiSpec, t string) spec.Type {
for _, tp := range api.Types {
@@ -69,32 +69,56 @@ func unread(r *bufio.Reader) error {
return r.UnreadRune()
}
func MatchStruct(api string) (info, structBody, service string, err error) {
r := regexp.MustCompile(typeRegex)
indexes := r.FindAllStringIndex(api, -1)
if len(indexes) == 0 {
return "", "", "", emptyStrcut
}
startIndexes := indexes[0]
endIndexes := indexes[len(indexes)-1]
func MatchStruct(api string) (*ApiStruct, error) {
var result ApiStruct
scanner := bufio.NewScanner(strings.NewReader(api))
var parseInfo = false
var parseImport = false
var parseType = false
var parseSevice = false
var segment string
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
info = api[:startIndexes[0]]
structBody = api[startIndexes[0]:endIndexes[len(endIndexes)-1]]
service = api[endIndexes[len(endIndexes)-1]:]
firstIIndex := strings.Index(info, "i")
if firstIIndex > 0 {
info = info[firstIIndex:]
}
lastServiceRightBraceIndex := strings.LastIndex(service, "}") + 1
var firstServiceIndex int
for index, char := range service {
if !isSpace(char) && !isNewline(char) {
firstServiceIndex = index
break
if line == "@doc(" {
parseInfo = true
}
if line == ")" && parseInfo {
parseInfo = false
result.Info = segment + ")"
segment = ""
continue
}
if strings.HasPrefix(line, "import") {
parseImport = true
}
if parseImport && (strings.HasPrefix(line, "type") || strings.HasPrefix(line, "@server") ||
strings.HasPrefix(line, "service")) {
parseImport = false
result.Imports = segment
segment = line + "\n"
continue
}
if strings.HasPrefix(line, "type") {
parseType = true
}
if strings.HasPrefix(line, "@server") || strings.HasPrefix(line, "service") {
if parseType {
parseType = false
result.StructBody = segment
segment = line + "\n"
continue
}
parseSevice = true
}
segment += scanner.Text() + "\n"
}
service = service[firstServiceIndex:lastServiceRightBraceIndex]
return
if !parseSevice {
return nil, errors.New("no service defined")
}
result.Service = segment
return &result, nil
}

View File

@@ -12,7 +12,7 @@ import (
)
const (
componentsTemplate = `// DO NOT EDIT, generated by goctl
componentsTemplate = `// Code generated by goctl. DO NOT EDIT.
{{.componentTypes}}
`

View File

@@ -57,24 +57,24 @@ var errJsonConvert = errors.New("json convert error")
{{.types}}
`
callInterfaceFunctionTemplate = `{{if .hasComment}}{{.comment}}
{{end}}{{.method}}(ctx context.Context,in *{{.pbRequest}}) {{if .hasResponse}}(*{{.pbResponse}},{{end}} error{{if .hasResponse}}){{end}}`
{{end}}{{.method}}(ctx context.Context,in *{{.pbRequest}}) (*{{.pbResponse}},error)`
callFunctionTemplate = `
{{if .hasComment}}{{.comment}}{{end}}
func (m *default{{.rpcServiceName}}) {{.method}}(ctx context.Context,in *{{.pbRequest}}) {{if .hasResponse}}(*{{.pbResponse}},{{end}} error{{if .hasResponse}}){{end}} {
func (m *default{{.rpcServiceName}}) {{.method}}(ctx context.Context,in *{{.pbRequest}}) (*{{.pbResponse}}, error) {
var request {{.package}}.{{.pbRequest}}
bts, err := jsonx.Marshal(in)
if err != nil {
return {{if .hasResponse}}nil, {{end}}errJsonConvert
return nil, errJsonConvert
}
err = jsonx.Unmarshal(bts, &request)
if err != nil {
return {{if .hasResponse}}nil, {{end}}errJsonConvert
return nil, errJsonConvert
}
client := {{.package}}.New{{.rpcServiceName}}Client(m.cli.Conn())
{{if .hasResponse}}resp, err := {{else}}_, err = {{end}}client.{{.method}}(ctx, &request)
{{if .hasResponse}}if err != nil{
resp, err := client.{{.method}}(ctx, &request)
if err != nil{
return nil, err
}
@@ -89,11 +89,7 @@ func (m *default{{.rpcServiceName}}) {{.method}}(ctx context.Context,in *{{.pbRe
return nil, errJsonConvert
}
return &ret, nil{{else}}if err != nil {
return err
}
return nil{{end}}
return &ret, nil
}
`
)
@@ -177,10 +173,6 @@ func (g *defaultRpcGenerator) getFuncs(service *parser.RpcService) ([]string, er
pkgName := file.Package
functions := make([]string, 0)
for _, method := range service.Funcs {
data, found := file.Strcuts[strings.ToLower(method.OutType)]
if found {
found = len(data.Field) > 0
}
var comment string
if len(method.Document) > 0 {
comment = method.Document[0]
@@ -191,7 +183,6 @@ func (g *defaultRpcGenerator) getFuncs(service *parser.RpcService) ([]string, er
"package": pkgName,
"pbRequest": method.InType,
"pbResponse": method.OutType,
"hasResponse": found,
"hasComment": len(method.Document) > 0,
"comment": comment,
})
@@ -205,26 +196,20 @@ func (g *defaultRpcGenerator) getFuncs(service *parser.RpcService) ([]string, er
}
func (g *defaultRpcGenerator) getInterfaceFuncs(service *parser.RpcService) ([]string, error) {
file := g.ast
functions := make([]string, 0)
for _, method := range service.Funcs {
data, found := file.Strcuts[strings.ToLower(method.OutType)]
if found {
found = len(data.Field) > 0
}
var comment string
if len(method.Document) > 0 {
comment = method.Document[0]
}
buffer, err := util.With("interfaceFn").Parse(callInterfaceFunctionTemplate).Execute(
map[string]interface{}{
"hasComment": len(method.Document) > 0,
"comment": comment,
"method": method.Name.Title(),
"pbRequest": method.InType,
"pbResponse": method.OutType,
"hasResponse": found,
"hasComment": len(method.Document) > 0,
"comment": comment,
"method": method.Name.Title(),
"pbRequest": method.InType,
"pbResponse": method.OutType,
})
if err != nil {
return nil, err

View File

@@ -445,9 +445,6 @@ func (a *PbAst) GenTypesCode() (string, error) {
}
func (s *Struct) genCode(containsTypeStatement bool) (string, error) {
if len(s.Field) == 0 {
return "", nil
}
fields := make([]string, 0)
for _, f := range s.Field {
var comment, doc string

View File

@@ -66,11 +66,11 @@ func buildDialOptions(opts ...ClientOption) []grpc.DialOption {
grpc.WithInsecure(),
grpc.WithBlock(),
WithUnaryClientInterceptors(
clientinterceptors.BreakerInterceptor,
clientinterceptors.TracingInterceptor,
clientinterceptors.DurationInterceptor,
clientinterceptors.BreakerInterceptor,
clientinterceptors.PromMetricInterceptor,
clientinterceptors.TimeoutInterceptor(clientOptions.Timeout),
clientinterceptors.TracingInterceptor,
),
}

View File

@@ -18,11 +18,13 @@ func DurationInterceptor(ctx context.Context, method string, req, reply interfac
start := timex.Now()
err := invoker(ctx, method, req, reply, cc, opts...)
if err != nil {
logx.WithDuration(timex.Since(start)).Infof("fail - %s - %v - %s", serverName, req, err.Error())
logx.WithContext(ctx).WithDuration(timex.Since(start)).Infof("fail - %s - %v - %s",
serverName, req, err.Error())
} else {
elapsed := timex.Since(start)
if elapsed > slowThreshold {
logx.WithDuration(elapsed).Slowf("[RPC] ok - slowcall - %s - %v - %v", serverName, req, reply)
logx.WithContext(ctx).WithDuration(elapsed).Slowf("[RPC] ok - slowcall - %s - %v - %v",
serverName, req, reply)
}
}

View File

@@ -17,6 +17,7 @@ type (
}
rpcServer struct {
name string
*baseRpcServer
}
)
@@ -40,6 +41,7 @@ func NewRpcServer(address string, opts ...ServerOption) Server {
}
func (s *rpcServer) SetName(name string) {
s.name = name
s.baseRpcServer.SetName(name)
}
@@ -50,6 +52,7 @@ func (s *rpcServer) Start(register RegisterFn) error {
}
unaryInterceptors := []grpc.UnaryServerInterceptor{
serverinterceptors.UnaryTracingInterceptor(s.name),
serverinterceptors.UnaryCrashInterceptor(),
serverinterceptors.UnaryStatInterceptor(s.metrics),
serverinterceptors.UnaryPromMetricInterceptor(),

View File

@@ -42,10 +42,11 @@ func logDuration(ctx context.Context, method string, req interface{}, duration t
}
content, err := json.Marshal(req)
if err != nil {
logx.Errorf("%s - %s", addr, err.Error())
logx.WithContext(ctx).Errorf("%s - %s", addr, err.Error())
} else if duration > serverSlowThreshold {
logx.WithDuration(duration).Slowf("[RPC] slowcall - %s - %s - %s", addr, method, string(content))
logx.WithContext(ctx).WithDuration(duration).Slowf("[RPC] slowcall - %s - %s - %s",
addr, method, string(content))
} else {
logx.WithDuration(duration).Infof("%s - %s - %s", addr, method, string(content))
logx.WithContext(ctx).WithDuration(duration).Infof("%s - %s - %s", addr, method, string(content))
}
}

View File

@@ -109,8 +109,6 @@ func setupInterceptors(server internal.Server, c RpcServerConf, metrics *stat.Me
time.Duration(c.Timeout) * time.Millisecond))
}
server.AddUnaryInterceptors(serverinterceptors.UnaryTracingInterceptor(c.Name))
if c.Auth {
authenticator, err := auth.NewAuthenticator(c.Redis.NewRedis(), c.Redis.Key, c.StrictControl)
if err != nil {