Compare commits

...

9 Commits

Author SHA1 Message Date
Keson
d12e9fa2d7 add model&rpc doc (#62) 2020-09-11 16:47:21 +08:00
miaogaolin
ce5961a7d0 fix goctl model (#61) 2020-09-11 16:46:45 +08:00
kingxt
e1d942a799 update doc (#64)
* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* update doc

* update doc

* remove update

Co-authored-by: kingxt <dream4kingxt@163.com>
2020-09-11 16:16:30 +08:00
kingxt
754e631dc4 update quick start (#63)
* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* update doc

* update readme.md

Co-authored-by: kingxt <dream4kingxt@163.com>
2020-09-11 16:06:04 +08:00
kevin
72aeac3fa9 add in-process cache doc 2020-09-11 15:30:20 +08:00
kingxt
1c3c8f4bbc add fast create api demo service (#59)
* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* add fast create api demo: goctl api new

* refactor

* refactor

Co-authored-by: kingxt <dream4kingxt@163.com>
2020-09-11 15:27:35 +08:00
Keson
17e6cfb7a9 quickly generating rpc demo service (#60)
* add execute files

* add protoc-osx

* add rpc generation

* add rpc generation

* add: rpc template generation

* add README.md

* format error

* reactor templatex.go

* update project.go & README.md

* add: quickly generate rpc service
2020-09-11 15:26:55 +08:00
kevin
0d151c17f8 update wechat image 2020-09-10 18:05:04 +08:00
miaogaolin
52990550fb fix GOMOD env fetch bug (#55) 2020-09-09 11:43:47 +08:00
23 changed files with 981 additions and 181 deletions

107
doc/collection.md Normal file
View File

@@ -0,0 +1,107 @@
# 通过 collection.Cache 进行缓存
go-zero微服务框架中提供了许多开箱即用的工具好的工具不仅能提升服务的性能而且还能提升代码的鲁棒性避免出错实现代码风格的统一方便他人阅读等等本系列文章将分别介绍go-zero框架中工具的使用及其实现原理
## 进程内缓存工具[collection.Cache](https://github.com/tal-tech/go-zero/tree/master/core/collection/cache.go)
在做服务器开发的时候相信都会遇到使用缓存的情况go-zero 提供的简单的缓存封装 **collection.Cache**,简单使用方式如下
```go
// 初始化 cache其中 WithLimit 可以指定最大缓存的数量
c, err := collection.NewCache(time.Minute, collection.WithLimit(10000))
if err != nil {
panic(err)
}
// 设置缓存
c.Set("key", user)
// 获取缓存ok是否存在
v, ok := c.Get("key")
// 删除缓存
c.Del("key")
// 获取缓存,如果 key 不存在的,则会调用 func 去生成缓存
v, err := c.Take("key", func() (interface{}, error) {
return user, nil
})
```
cache 实现的建的功能包括
* 缓存自动失效,可以指定过期时间
* 缓存大小限制,可以指定缓存个数
* 缓存增删改
* 缓存命中率统计
* 并发安全
* 缓存击穿
实现原理:
Cache 自动失效,是采用 TimingWheel(https://github.com/tal-tech/go-zero/blob/master/core/collection/timingwheel.go) 进行管理的
``` go
timingWheel, err := NewTimingWheel(time.Second, slots, func(k, v interface{}) {
key, ok := k.(string)
if !ok {
return
}
cache.Del(key)
})
```
Cache 大小限制,是采用 LRU 淘汰策略,在新增缓存的时候会去检查是否已经超出过限制,具体代码在 keyLru 中实现
``` go
func (klru *keyLru) add(key string) {
if elem, ok := klru.elements[key]; ok {
klru.evicts.MoveToFront(elem)
return
}
// Add new item
elem := klru.evicts.PushFront(key)
klru.elements[key] = elem
// Verify size not exceeded
if klru.evicts.Len() > klru.limit {
klru.removeOldest()
}
}
```
Cache 的命中率统计,是在代码中实现 cacheStat,在缓存命中丢失的时候自动统计,并且会定时打印使用的命中率, qps 等状态.
打印的具体效果如下
```go
cache(proc) - qpm: 2, hit_ratio: 50.0%, elements: 0, hit: 1, miss: 1
```
缓存击穿包含是使用 syncx.SharedCalls(https://github.com/tal-tech/go-zero/blob/master/core/syncx/sharedcalls.go) 进行实现的,就是将同时请求同一个 key 的请求, 关于 sharedcalls 后续会继续补充。 相关具体实现是在:
```go
func (c *Cache) Take(key string, fetch func() (interface{}, error)) (interface{}, error) {
val, fresh, err := c.barrier.DoEx(key, func() (interface{}, error) {
v, e := fetch()
if e != nil {
return nil, e
}
c.Set(key, v)
return v, nil
})
if err != nil {
return nil, err
}
if fresh {
c.stats.IncrementMiss()
return val, nil
} else {
// got the result from previous ongoing query
c.stats.IncrementHit()
}
return val, nil
}
```
本文主要介绍了go-zero框架中的 Cache 工具,在实际的项目中非常实用。用好工具对于提升服务性能和开发效率都有很大的帮助,希望本篇文章能给大家带来一些收获。

274
doc/goctl-model-sql.md Normal file
View File

@@ -0,0 +1,274 @@
# Goctl Model
goctl model 为go-zero下的工具模块中的组件之一目前支持识别mysql ddl进行model层代码生成通过命令行或者idea插件即将支持可以有选择地生成带redis cache或者不带redis cache的代码逻辑。
# 快速开始
* 通过ddl生成
```shell script
$ goctl model mysql ddl -src="./sql/user.sql" -dir="./sql/model" -c=true
```
执行上述命令后即可快速生成CURD代码。
```
model
│   ├── error.go
│   └── usermodel.go
```
* 通过datasource生成
```shell script
$ goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="table1,table2" -dir="./model"
```
* 生成代码示例
``` go
package model
import (
"database/sql"
"fmt"
"strings"
"time"
"github.com/tal-tech/go-zero/core/stores/cache"
"github.com/tal-tech/go-zero/core/stores/sqlc"
"github.com/tal-tech/go-zero/core/stores/sqlx"
"github.com/tal-tech/go-zero/core/stringx"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/builderx"
)
var (
userFieldNames = builderx.FieldNames(&User{})
userRows = strings.Join(userFieldNames, ",")
userRowsExpectAutoSet = strings.Join(stringx.Remove(userFieldNames, "id", "create_time", "update_time"), ",")
userRowsWithPlaceHolder = strings.Join(stringx.Remove(userFieldNames, "id", "create_time", "update_time"), "=?,") + "=?"
cacheUserMobilePrefix = "cache#User#mobile#"
cacheUserIdPrefix = "cache#User#id#"
cacheUserNamePrefix = "cache#User#name#"
)
type (
UserModel struct {
sqlc.CachedConn
table string
}
User struct {
Id int64 `db:"id"`
Name string `db:"name"` // 用户名称
Password string `db:"password"` // 用户密码
Mobile string `db:"mobile"` // 手机号
Gender string `db:"gender"` // 男|女|未公开
Nickname string `db:"nickname"` // 用户昵称
CreateTime time.Time `db:"create_time"`
UpdateTime time.Time `db:"update_time"`
}
)
func NewUserModel(conn sqlx.SqlConn, c cache.CacheConf, table string) *UserModel {
return &UserModel{
CachedConn: sqlc.NewConn(conn, c),
table: table,
}
}
func (m *UserModel) Insert(data User) (sql.Result, error) {
query := `insert into ` + m.table + `(` + userRowsExpectAutoSet + `) value (?, ?, ?, ?, ?)`
return m.ExecNoCache(query, data.Name, data.Password, data.Mobile, data.Gender, data.Nickname)
}
func (m *UserModel) FindOne(id int64) (*User, error) {
userIdKey := fmt.Sprintf("%s%v", cacheUserIdPrefix, id)
var resp User
err := m.QueryRow(&resp, userIdKey, func(conn sqlx.SqlConn, v interface{}) error {
query := `select ` + userRows + ` from ` + m.table + ` where id = ? limit 1`
return conn.QueryRow(v, query, id)
})
switch err {
case nil:
return &resp, nil
case sqlc.ErrNotFound:
return nil, ErrNotFound
default:
return nil, err
}
}
func (m *UserModel) FindOneByName(name string) (*User, error) {
userNameKey := fmt.Sprintf("%s%v", cacheUserNamePrefix, name)
var resp User
err := m.QueryRowIndex(&resp, userNameKey, func(primary interface{}) string {
return fmt.Sprintf("%s%v", cacheUserIdPrefix, primary)
}, func(conn sqlx.SqlConn, v interface{}) (i interface{}, e error) {
query := `select ` + userRows + ` from ` + m.table + ` where name = ? limit 1`
if err := conn.QueryRow(&resp, query, name); err != nil {
return nil, err
}
return resp.Id, nil
}, func(conn sqlx.SqlConn, v, primary interface{}) error {
query := `select ` + userRows + ` from ` + m.table + ` where id = ? limit 1`
return conn.QueryRow(v, query, primary)
})
switch err {
case nil:
return &resp, nil
case sqlc.ErrNotFound:
return nil, ErrNotFound
default:
return nil, err
}
}
func (m *UserModel) FindOneByMobile(mobile string) (*User, error) {
userMobileKey := fmt.Sprintf("%s%v", cacheUserMobilePrefix, mobile)
var resp User
err := m.QueryRowIndex(&resp, userMobileKey, func(primary interface{}) string {
return fmt.Sprintf("%s%v", cacheUserIdPrefix, primary)
}, func(conn sqlx.SqlConn, v interface{}) (i interface{}, e error) {
query := `select ` + userRows + ` from ` + m.table + ` where mobile = ? limit 1`
if err := conn.QueryRow(&resp, query, mobile); err != nil {
return nil, err
}
return resp.Id, nil
}, func(conn sqlx.SqlConn, v, primary interface{}) error {
query := `select ` + userRows + ` from ` + m.table + ` where id = ? limit 1`
return conn.QueryRow(v, query, primary)
})
switch err {
case nil:
return &resp, nil
case sqlc.ErrNotFound:
return nil, ErrNotFound
default:
return nil, err
}
}
func (m *UserModel) Update(data User) error {
userIdKey := fmt.Sprintf("%s%v", cacheUserIdPrefix, data.Id)
_, err := m.Exec(func(conn sqlx.SqlConn) (result sql.Result, err error) {
query := `update ` + m.table + ` set ` + userRowsWithPlaceHolder + ` where id = ?`
return conn.Exec(query, data.Name, data.Password, data.Mobile, data.Gender, data.Nickname, data.Id)
}, userIdKey)
return err
}
func (m *UserModel) Delete(id int64) error {
data, err := m.FindOne(id)
if err != nil {
return err
}
userIdKey := fmt.Sprintf("%s%v", cacheUserIdPrefix, id)
userNameKey := fmt.Sprintf("%s%v", cacheUserNamePrefix, data.Name)
userMobileKey := fmt.Sprintf("%s%v", cacheUserMobilePrefix, data.Mobile)
_, err = m.Exec(func(conn sqlx.SqlConn) (result sql.Result, err error) {
query := `delete from ` + m.table + ` where id = ?`
return conn.Exec(query, id)
}, userIdKey, userNameKey, userMobileKey)
return err
}
```
# 用法
```
$ goctl model mysql -h
```
```
NAME:
goctl model mysql - generate mysql model"
USAGE:
goctl model mysql command [command options] [arguments...]
COMMANDS:
ddl generate mysql model from ddl"
datasource generate model from datasource"
OPTIONS:
--help, -h show help
```
# 生成规则
* 默认规则
我们默认用户在建表时会创建createTime、updateTime字段(忽略大小写、下划线命名风格)且默认值均为`CURRENT_TIMESTAMP`而updateTime支持`ON UPDATE CURRENT_TIMESTAMP`,对于这两个字段生成`insert`、`update`时会被移除,不在赋值范畴内,当然,如果你不需要这两个字段那也无大碍。
* 带缓存模式
* ddl
```shell script
$ goctl model mysql -src={filename} -dir={dir} -cache=true
```
* datasource
```shell script
$ goctl model mysql datasource -url={datasource} -table={tables} -dir={dir} -cache=true
```
目前仅支持redis缓存如果选择带缓存模式即生成的`FindOne(ByXxx)`&`Delete`代码会生成带缓存逻辑的代码目前仅支持单索引字段除全文索引外对于联合索引我们默认认为不需要带缓存且不属于通用型代码因此没有放在代码生成行列如example中user表中的`id`、`name`、`mobile`字段均属于单字段索引。
* 不带缓存模式
* ddl
```shell script
$ goctl model -src={filename} -dir={dir}
```
* datasource
```shell script
$ goctl model mysql datasource -url={datasource} -table={tables} -dir={dir}
```
or
* ddl
```shell script
$ goctl model -src={filename} -dir={dir} -cache=false
```
* datasource
```shell script
$ goctl model mysql datasource -url={datasource} -table={tables} -dir={dir} -cache=false
```
生成代码仅基本的CURD结构。
# 缓存
对于缓存这一块我选择用一问一答的形式进行罗列。我想这样能够更清晰的描述model中缓存的功能。
* 缓存会缓存哪些信息?
对于主键字段缓存,会缓存整个结构体信息,而对于单索引字段(除全文索引)则缓存主键字段值。
* 数据有更新(`update`)操作会清空缓存吗?
但仅清空主键缓存的信息why这里就不做详细赘述了。
* 为什么不按照单索引字段生成`updateByXxx`和`deleteByXxx`的代码?
理论上是没任何问题但是我们认为对于model层的数据操作均是以整个结构体为单位包括查询我不建议只查询某部分字段不反对否则我们的缓存就没有意义了。
* 为什么不支持`findPageLimit`、`findAll`这么模式代码生层?
目前我认为除了基本的CURD外其他的代码均属于<i>业务型</i>代码,这个我觉得开发人员根据业务需要进行编写更好。
# QA
* goctl model除了命令行模式支持插件模式吗
很快支持idea插件。

223
doc/goctl-rpc.md Normal file
View File

@@ -0,0 +1,223 @@
# Rpc Generation
Goctl Rpc是`goctl`脚手架下的一个rpc服务代码生成模块支持proto模板生成和rpc服务代码生成通过此工具生成代码你只需要关注业务逻辑编写而不用去编写一些重复性的代码。这使得我们把精力重心放在业务上从而加快了开发效率且降低了代码出错率。
# 特性
* 简单易用
* 快速提升开发效率
* 出错率低
# 快速开始
### 方式一快速生成greet服务
通过命令 `goctl rpc new ${servieName}`生成
如生成greet rpc服务
```shell script
$ goctl rpc new greet
```
执行后代码结构如下:
```golang
└── greet
├── etc
│   └── greet.yaml
├── go.mod
├── go.sum
├── greet
│   ├── greet.go
│   ├── greet_mock.go
│   └── types.go
├── greet.go
├── greet.proto
├── internal
│   ├── config
│   │   └── config.go
│   ├── logic
│   │   └── pinglogic.go
│   ├── server
│   │   └── greetserver.go
│   └── svc
│   └── servicecontext.go
└── pb
└── greet.pb.go
```
rpc一键生成常见问题解决见 <a href="#常见问题解决">常见问题解决</a>
### 方式二通过指定proto生成rpc服务
* 生成proto模板
```shell script
$ goctl rpc template -o=user.proto
```
```golang
syntax = "proto3";
package remote;
message Request {
// 用户名
string username = 1;
// 用户密码
string password = 2;
}
message Response {
// 用户名称
string name = 1;
// 用户性别
string gender = 2;
}
service User {
// 登录
rpc Login(Request)returns(Response);
}
```
* 生成rpc服务代码
```
$ goctl rpc proto -src=user.proto
```
代码tree
```
user
├── etc
│   └── user.json
├── internal
│   ├── config
│   │   └── config.go
│   ├── handler
│   │   ├── loginhandler.go
│   ├── logic
│   │   └── loginlogic.go
│   └── svc
│   └── servicecontext.go
├── pb
│   └── user.pb.go
├── shared
│   ├── mockusermodel.go
│   ├── types.go
│   └── usermodel.go
├── user.go
└── user.proto
```
# 准备工作
* 安装了go环境
* 安装了protoc&protoc-gen-go并且已经设置环境变量
* mockgen(可选,将移除)
* 更多问题请见 <a href="#注意事项">注意事项</a>
# 用法
### rpc服务生成用法
```shell script
$ goctl rpc proto -h
```
```shell script
NAME:
goctl rpc proto - generate rpc from proto
USAGE:
goctl rpc proto [command options] [arguments...]
OPTIONS:
--src value, -s value the file path of the proto source file
--dir value, -d value the target path of the code,default path is "${pwd}". [option]
--service value, --srv value the name of rpc service. [option]
--shared[已废弃] value the dir of the shared file,default path is "${pwd}/shared. [option]"
--idea whether the command execution environment is from idea plugin. [option]
```
### 参数说明
* --src 必填proto数据源目前暂时支持单个proto文件生成这里不支持不建议外部依赖
* --dir 非必填默认为proto文件所在目录生成代码的目标目录
* --service 服务名称非必填默认为proto文件所在目录名称但是如果proto所在目录为一下结构
```shell script
user
├── cmd
│   └── rpc
│   └── user.proto
```
则服务名称亦为user而非proto所在文件夹名称了这里推荐使用这种结构可以方便在同一个服务名下建立不同类型的服务(api、rpc、mq等),便于代码管理与维护。
* --shared[⚠️已废弃] 非必填,默认为$dir(xxx.proto)/sharedrpc client逻辑代码存放目录。
> 注意这里的shared文件夹名称将会是代码中的package名称。
* --idea 非必填是否为idea插件中执行保留字段终端执行可以忽略
# 开发人员需要做什么
关注业务代码编写将重复性、与业务无关的工作交给goctl生成好rpc服务代码后开饭人员仅需要修改
* 服务中的配置文件编写(etc/xx.json、internal/config/config.go)
* 服务中业务逻辑编写(internal/logic/xxlogic.go)
* 服务中资源上下文的编写(internal/svc/servicecontext.go)
# 扩展
对于需要进行rpc mock的开发人员在安装了`mockgen`工具的前提下可以在rpc的shared文件中生成好对应的mock文件。
# 注意事项
* `google.golang.org/grpc`需要降级到v1.26.0,且protoc-gen-go版本不能高于v1.3.2see [https://github.com/grpc/grpc-go/issues/3347](https://github.com/grpc/grpc-go/issues/3347))即
```
replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
```
* proto不支持暂多文件同时生成
* proto不支持外部依赖包引入message不支持inline
* 目前main文件、shared文件、handler文件会被强制覆盖而和开发人员手动需要编写的则不会覆盖生成这一类在代码头部均有
```shell script
// Code generated by goctl. DO NOT EDIT!
// Source: xxx.proto
```
的标识,请注意不要将也写业务性代码写在里面。
# 常见问题解决(go mod工程)
* 错误一:
```golang
pb/xx.pb.go:220:7: undefined: grpc.ClientConnInterface
pb/xx.pb.go:224:11: undefined: grpc.SupportPackageIsVersion6
pb/xx.pb.go:234:5: undefined: grpc.ClientConnInterface
pb/xx.pb.go:237:24: undefined: grpc.ClientConnInterface
```
解决方法:请将`protoc-gen-go`版本降至v1.3.2及一下
* 错误二:
```golang
# go.etcd.io/etcd/clientv3/balancer/picker
../../../go/pkg/mod/go.etcd.io/etcd@v0.0.0-20200402134248-51bdeb39e698/clientv3/balancer/picker/err.go:25:9: cannot use &errPicker literal (type *errPicker) as type Picker in return argument:*errPicker does not implement Picker (wrong type for Pick method)
have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
want Pick(balancer.PickInfo) (balancer.PickResult, error)
../../../go/pkg/mod/go.etcd.io/etcd@v0.0.0-20200402134248-51bdeb39e698/clientv3/balancer/picker/roundrobin_balanced.go:33:9: cannot use &rrBalanced literal (type *rrBalanced) as type Picker in return argument:
*rrBalanced does not implement Picker (wrong type for Pick method)
have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
want Pick(balancer.PickInfo) (balancer.PickResult, error)
#github.com/tal-tech/go-zero/rpcx/internal/balancer/p2c
../../../go/pkg/mod/github.com/tal-tech/go-zero@v1.0.12/rpcx/internal/balancer/p2c/p2c.go:41:32: not enough arguments in call to base.NewBalancerBuilder
have (string, *p2cPickerBuilder)
want (string, base.PickerBuilder, base.Config)
../../../go/pkg/mod/github.com/tal-tech/go-zero@v1.0.12/rpcx/internal/balancer/p2c/p2c.go:58:9: cannot use &p2cPicker literal (type *p2cPicker) as type balancer.Picker in return argument:
*p2cPicker does not implement balancer.Picker (wrong type for Pick method)
have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
want Pick(balancer.PickInfo) (balancer.PickResult, error)
```
解决方法:
```golang
replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
```

View File

@@ -7,6 +7,11 @@
* 生成MongoDB CURD+Cache
## goctl使用说明
#### 快速生成服务
* api: goctl api new xxxx
* rpc: goctl rpc new xxxx
#### goctl参数说明
`goctl api [go/java/ts] [-api user/user.api] [-dir ./src]`
@@ -17,17 +22,10 @@
> -dir 自定义生成目录
#### 保持goctl总是最新版
第一次运行会在~/.goctl里增加下面两行
```
url = http://47.97.184.41:7777/
```
#### API 语法说明
```
``` golang
info(
title: doc title
desc: >
@@ -123,7 +121,9 @@ service user-api {
)
head /api/ping()
}
```
1. info部分描述了api基本信息比如Authapi是哪个用途。
2. type部分type类型声明和golang语法兼容。
3. service部分service代表一组服务一个服务可以由多组名称相同的service组成可以针对每一组service配置jwt和auth认证另外通过folder属性可以指定service生成所在子目录。

Binary file not shown.

Before

Width:  |  Height:  |  Size: 148 KiB

After

Width:  |  Height:  |  Size: 125 KiB

View File

@@ -107,61 +107,10 @@ go get -u github.com/tal-tech/go-zero
确保goctl可执行
2. 定义API文件比如greet.api可以在vs code里安装`goctl`插件支持api语法
```go
type Request struct {
Name string `path:"name,options=you|me"` // 框架自动验证请求参数是否合法
}
2. 快速生成api服务
type Response struct {
Message string `json:"message"`
}
service greet-api {
@server(
handler: GreetHandler
)
get /greet/from/:name(Request) returns (Response);
}
```
也可以通过goctl生成api模本文件命令如下
```shell
goctl api -o greet.api
```
3. 生成go服务端代码
```shell
goctl api go -api greet.api -dir greet
```
生成的文件结构如下:
```
├── greet
│   ├── etc
│   │   └── greet-api.yaml // 配置文件
│   ├── greet.go // main文件
│   └── internal
│   ├── config
│   │   └── config.go // 配置定义
│   ├── handler
│   │   ├── greethandler.go // get/put/post/delete等路由定义文件
│   │   └── routes.go // 路由列表
│   ├── logic
│   │   └── greetlogic.go // 请求逻辑处理文件
│   ├── svc
│   │   └── servicecontext.go // 请求上下文可以传入mysql, redis等依赖
│   └── types
│   └── types.go // 请求、返回等类型定义
└── greet.api // api描述文件
```
生成的代码可以直接运行:
```shell
goctl api new greet
cd greet
go run greet.go -f etc/greet-api.yaml
```
@@ -182,10 +131,11 @@ Content-Length: 0
编写业务代码:
* api文件定义了服务对外暴露的路由可参考[api规范](https://github.com/tal-tech/go-zero/blob/master/doc/goctl.md)
* 可以在servicecontext.go里面传递依赖给logic比如mysql, redis等
* 在api定义的get/post/put/delete等请求对应的logic里增加业务处理逻辑
4. 可以根据api文件生成前端需要的Java, TypeScript, Dart, JavaScript代码
3. 可以根据api文件生成前端需要的Java, TypeScript, Dart, JavaScript代码
```shell
goctl api java -api greet.api -dir greet
@@ -206,6 +156,7 @@ Content-Length: 0
* [goctl使用帮助](doc/goctl.md)
* [通过MapReduce降低服务响应时间](doc/mapreduce.md)
* [关键字替换和敏感词过滤工具](doc/keywords.md)
* [进程内缓存使用方法](doc/collection.md)
## 9. 微信交流群

View File

@@ -36,6 +36,10 @@ func GoCommand(c *cli.Context) error {
return errors.New("missing -dir")
}
return DoGenProject(apiFile, dir)
}
func DoGenProject(apiFile, dir string) error {
p, err := parser.NewParser(apiFile)
if err != nil {
return err

View File

@@ -0,0 +1,58 @@
package new
import (
"os"
"path/filepath"
"text/template"
"github.com/tal-tech/go-zero/tools/goctl/api/gogen"
"github.com/urfave/cli"
)
const apiTemplate = `
type Request struct {
Name string ` + "`" + `path:"name,options=you|me"` + "`" + ` // 框架自动验证请求参数是否合法
}
type Response struct {
Message string ` + "`" + `json:"message"` + "`" + `
}
service {{.name}}-api {
@server(
handler: GreetHandler
)
get /greet/from/:name(Request) returns (Response);
}
`
func NewService(c *cli.Context) error {
args := c.Args()
name := "greet"
if len(args) > 0 {
name = args.First()
}
location := name
err := os.MkdirAll(location, os.ModePerm)
if err != nil {
return err
}
filename := name + ".api"
apiFilePath := filepath.Join(location, filename)
fp, err := os.Create(apiFilePath)
if err != nil {
return err
}
defer fp.Close()
t := template.Must(template.New("template").Parse(apiTemplate))
if err := t.Execute(fp, map[string]string{
"name": name,
}); err != nil {
return err
}
err = gogen.DoGenProject(apiFilePath, location)
return err
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/tal-tech/go-zero/tools/goctl/api/gogen"
"github.com/tal-tech/go-zero/tools/goctl/api/javagen"
"github.com/tal-tech/go-zero/tools/goctl/api/ktgen"
"github.com/tal-tech/go-zero/tools/goctl/api/new"
"github.com/tal-tech/go-zero/tools/goctl/api/tsgen"
"github.com/tal-tech/go-zero/tools/goctl/api/validate"
"github.com/tal-tech/go-zero/tools/goctl/configgen"
@@ -36,6 +37,11 @@ var (
},
Action: apigen.ApiCommand,
Subcommands: []cli.Command{
{
Name: "new",
Usage: "fast create api service",
Action: new.NewService,
},
{
Name: "format",
Usage: "format api files",
@@ -193,6 +199,17 @@ var (
Name: "rpc",
Usage: "generate rpc code",
Subcommands: []cli.Command{
{
Name: "new",
Usage: `generate rpc demo service`,
Flags: []cli.Flag{
cli.BoolFlag{
Name: "idea",
Usage: "whether the command execution environment is from idea plugin. [option]",
},
},
Action: rpc.RpcNew,
},
{
Name: "template",
Usage: `generate proto template`,
@@ -224,10 +241,6 @@ var (
Name: "service, srv",
Usage: `the name of rpc service. [option]`,
},
cli.StringFlag{
Name: "shared",
Usage: `the dir of the shared file,default path is "${pwd}/shared. [option]`,
},
cli.BoolFlag{
Name: "idea",
Usage: "whether the command execution environment is from idea plugin. [option]",

View File

@@ -22,9 +22,6 @@ goctl model 为go-zero下的工具模块中的组件之一目前支持识别m
```shell script
$ goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="table1,table2" -dir="./model"
```
> 详情用法请参考[example](https://github.com/tal-tech/go-zero/tree/master/tools/goctl/model/sql/example)
* 生成代码示例

View File

@@ -22,7 +22,7 @@ func genDelete(table Table, withCache bool) (string, error) {
}
var containsIndexCache = false
for _, item := range table.Fields {
if item.IsKey && !item.IsPrimaryKey {
if item.IsUniqueKey {
containsIndexCache = true
break
}

View File

@@ -14,7 +14,7 @@ func genFineOneByField(table Table, withCache bool) (string, error) {
var list []string
camelTableName := table.Name.ToCamel()
for _, field := range table.Fields {
if field.IsPrimaryKey || !field.IsKey {
if field.IsPrimaryKey || !field.IsUniqueKey {
continue
}
camelFieldName := field.Name.ToCamel()

View File

@@ -29,22 +29,21 @@ func genCacheKeys(table parser.Table) (map[string]Key, error) {
camelTableName := table.Name.ToCamel()
lowerStartCamelTableName := stringx.From(camelTableName).UnTitle()
for _, field := range fields {
if !field.IsKey {
continue
}
camelFieldName := field.Name.ToCamel()
lowerStartCamelFieldName := stringx.From(camelFieldName).UnTitle()
left := fmt.Sprintf("cache%s%sPrefix", camelTableName, camelFieldName)
right := fmt.Sprintf("cache#%s#%s#", camelTableName, lowerStartCamelFieldName)
variable := fmt.Sprintf("%s%sKey", lowerStartCamelTableName, camelFieldName)
m[field.Name.Source()] = Key{
VarExpression: fmt.Sprintf(`%s = "%s"`, left, right),
Left: left,
Right: right,
Variable: variable,
KeyExpression: fmt.Sprintf(`%s := fmt.Sprintf("%s%s", %s,%s)`, variable, "%s", "%v", left, lowerStartCamelFieldName),
DataKeyExpression: fmt.Sprintf(`%s := fmt.Sprintf("%s%s",%s, data.%s)`, variable, "%s", "%v", left, camelFieldName),
RespKeyExpression: fmt.Sprintf(`%s := fmt.Sprintf("%s%s", %s,resp.%s)`, variable, "%s", "%v", left, camelFieldName),
if field.IsUniqueKey || field.IsPrimaryKey {
camelFieldName := field.Name.ToCamel()
lowerStartCamelFieldName := stringx.From(camelFieldName).UnTitle()
left := fmt.Sprintf("cache%s%sPrefix", camelTableName, camelFieldName)
right := fmt.Sprintf("cache#%s#%s#", camelTableName, lowerStartCamelFieldName)
variable := fmt.Sprintf("%s%sKey", lowerStartCamelTableName, camelFieldName)
m[field.Name.Source()] = Key{
VarExpression: fmt.Sprintf(`%s = "%s"`, left, right),
Left: left,
Right: right,
Variable: variable,
KeyExpression: fmt.Sprintf(`%s := fmt.Sprintf("%s%s", %s,%s)`, variable, "%s", "%v", left, lowerStartCamelFieldName),
DataKeyExpression: fmt.Sprintf(`%s := fmt.Sprintf("%s%s",%s, data.%s)`, variable, "%s", "%v", left, camelFieldName),
RespKeyExpression: fmt.Sprintf(`%s := fmt.Sprintf("%s%s", %s,resp.%s)`, variable, "%s", "%v", left, camelFieldName),
}
}
}
return m, nil

View File

@@ -36,6 +36,7 @@ type (
DataType string
IsKey bool
IsPrimaryKey bool
IsUniqueKey bool
Comment string
}
@@ -124,6 +125,7 @@ func Parse(ddl string) (*Table, error) {
if ok {
field.IsKey = true
field.IsPrimaryKey = key == primary
field.IsUniqueKey = key == unique
if field.IsPrimaryKey {
primaryKey.Field = field
if column.Type.Autoincrement {

View File

@@ -1,5 +1,10 @@
# Change log
# 2020-08-29
* rpc greet服务一键生成
* 修复相对路径生成rpc服务package引入错误bug
* 移除`--shared`参数
# 2020-08-29
* 新增支持windows生成

View File

@@ -8,74 +8,118 @@ Goctl Rpc是`goctl`脚手架下的一个rpc服务代码生成模块支持prot
# 快速开始
### 生成proto模板
### 方式一快速生成greet服务
```shell script
$ goctl rpc template -o=user.proto
```
通过命令 `goctl rpc new ${servieName}`生成
```golang
syntax = "proto3";
如生成greet rpc服务
package remote;
```shell script
$ goctl rpc new greet
```
message Request {
// 用户名
string username = 1;
// 用户密码
string password = 2;
}
执行后代码结构如下:
message Response {
// 用户名称
string name = 1;
// 用户性别
string gender = 2;
}
service User {
// 登录
rpc Login(Request)returns(Response);
}
```
### 生成rpc服务代码
生成user rpc服务
```
$ goctl rpc proto -src=user.proto
```
代码tree
```
user
```golang
└── greet
├── etc
│   └── user.json
│   └── greet.yaml
├── go.mod
├── go.sum
├── greet
│   ├── greet.go
│   ├── greet_mock.go
│   └── types.go
├── greet.go
├── greet.proto
├── internal
│   ├── config
│   │   └── config.go
│   ├── handler
│   │   ├── loginhandler.go
│   ├── logic
│   │   └── loginlogic.go
│   │   └── pinglogic.go
│   ├── server
│   │   └── greetserver.go
│   └── svc
│   └── servicecontext.go
── pb
│   └── user.pb.go
├── shared
│   ├── mockusermodel.go
│   ├── types.go
│   └── usermodel.go
├── user.go
└── user.proto
── pb
└── greet.pb.go
```
```
rpc一键生成常见问题解决见 <a href="#常见问题解决">常见问题解决</a>
### 方式二通过指定proto生成rpc服务
* 生成proto模板
```shell script
$ goctl rpc template -o=user.proto
```
```golang
syntax = "proto3";
package remote;
message Request {
// 用户名
string username = 1;
// 用户密码
string password = 2;
}
message Response {
// 用户名称
string name = 1;
// 用户性别
string gender = 2;
}
service User {
// 登录
rpc Login(Request)returns(Response);
}
```
* 生成rpc服务代码
```
$ goctl rpc proto -src=user.proto
```
代码tree
```
user
├── etc
│   └── user.json
├── internal
│   ├── config
│   │   └── config.go
│   ├── handler
│   │   ├── loginhandler.go
│   ├── logic
│   │   └── loginlogic.go
│   └── svc
│   └── servicecontext.go
├── pb
│   └── user.pb.go
├── shared
│   ├── mockusermodel.go
│   ├── types.go
│   └── usermodel.go
├── user.go
└── user.proto
```
# 准备工作
* 安装了go环境
* 安装了protoc&protoc-gen-go并且已经设置环境变量
* mockgen(可选)
* mockgen(可选,将移除)
* 更多问题请见 <a href="#注意事项">注意事项</a>
# 用法
### rpc服务生成用法
```shell script
$ goctl rpc proto -h
```
@@ -91,27 +135,28 @@ OPTIONS:
--src value, -s value the file path of the proto source file
--dir value, -d value the target path of the code,default path is "${pwd}". [option]
--service value, --srv value the name of rpc service. [option]
--shared value the dir of the shared file,default path is "${pwd}/shared. [option]"
--shared[已废弃] value the dir of the shared file,default path is "${pwd}/shared. [option]"
--idea whether the command execution environment is from idea plugin. [option]
```
* 参数说明
* --src 必填proto数据源目前暂时支持单个proto文件生成这里不支持不建议外部依赖
* --dir 非必填默认为proto文件所在目录生成代码的目标目录
* --service 服务名称非必填默认为proto文件所在目录名称但是如果proto所在目录为一下结构
```shell script
user
├── cmd
│   └── rpc
│   └── user.proto
```
则服务名称亦为user而非proto所在文件夹名称了这里推荐使用这种结构可以方便在同一个服务名下建立不同类型的服务(api、rpc、mq等),便于代码管理与维护。
* --shared 非必填,默认为$dir(xxx.proto)/sharedrpc client逻辑代码存放目录
> 注意这里的shared文件夹名称将会是代码中的package名称。
* --idea 非必填是否为idea插件中执行保留字段终端执行可以忽略
### 参数说明
* --src 必填proto数据源目前暂时支持单个proto文件生成这里不支持不建议外部依赖
* --dir 非必填默认为proto文件所在目录生成代码的目标目录
* --service 服务名称非必填默认为proto文件所在目录名称但是如果proto所在目录为一下结构
```shell script
user
├── cmd
│   └── rpc
│   └── user.proto
```
则服务名称亦为user而非proto所在文件夹名称了这里推荐使用这种结构可以方便在同一个服务名下建立不同类型的服务(api、rpc、mq等),便于代码管理与维护
* --shared[⚠️已废弃] 非必填,默认为$dir(xxx.proto)/sharedrpc client逻辑代码存放目录。
> 注意这里的shared文件夹名称将会是代码中的package名称。
* --idea 非必填是否为idea插件中执行保留字段终端执行可以忽略
# 开发人员需要做什么
@@ -124,6 +169,10 @@ OPTIONS:
对于需要进行rpc mock的开发人员在安装了`mockgen`工具的前提下可以在rpc的shared文件中生成好对应的mock文件。
# 注意事项
* `google.golang.org/grpc`需要降级到v1.26.0,且protoc-gen-go版本不能高于v1.3.2see [https://github.com/grpc/grpc-go/issues/3347](https://github.com/grpc/grpc-go/issues/3347))即
```
replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
```
* proto不支持暂多文件同时生成
* proto不支持外部依赖包引入message不支持inline
* 目前main文件、shared文件、handler文件会被强制覆盖而和开发人员手动需要编写的则不会覆盖生成这一类在代码头部均有
@@ -133,8 +182,42 @@ OPTIONS:
```
的标识,请注意不要将也写业务性代码写在里面。
# 常见问题解决(go mod工程)
* 错误一:
```golang
pb/xx.pb.go:220:7: undefined: grpc.ClientConnInterface
pb/xx.pb.go:224:11: undefined: grpc.SupportPackageIsVersion6
pb/xx.pb.go:234:5: undefined: grpc.ClientConnInterface
pb/xx.pb.go:237:24: undefined: grpc.ClientConnInterface
```
解决方法:请将`protoc-gen-go`版本降至v1.3.2及一下
* 错误二:
```golang
# go.etcd.io/etcd/clientv3/balancer/picker
../../../go/pkg/mod/go.etcd.io/etcd@v0.0.0-20200402134248-51bdeb39e698/clientv3/balancer/picker/err.go:25:9: cannot use &errPicker literal (type *errPicker) as type Picker in return argument:*errPicker does not implement Picker (wrong type for Pick method)
have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
want Pick(balancer.PickInfo) (balancer.PickResult, error)
../../../go/pkg/mod/go.etcd.io/etcd@v0.0.0-20200402134248-51bdeb39e698/clientv3/balancer/picker/roundrobin_balanced.go:33:9: cannot use &rrBalanced literal (type *rrBalanced) as type Picker in return argument:
*rrBalanced does not implement Picker (wrong type for Pick method)
have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
want Pick(balancer.PickInfo) (balancer.PickResult, error)
#github.com/tal-tech/go-zero/rpcx/internal/balancer/p2c
../../../go/pkg/mod/github.com/tal-tech/go-zero@v1.0.12/rpcx/internal/balancer/p2c/p2c.go:41:32: not enough arguments in call to base.NewBalancerBuilder
have (string, *p2cPickerBuilder)
want (string, base.PickerBuilder, base.Config)
../../../go/pkg/mod/github.com/tal-tech/go-zero@v1.0.12/rpcx/internal/balancer/p2c/p2c.go:58:9: cannot use &p2cPicker literal (type *p2cPicker) as type balancer.Picker in return argument:
*p2cPicker does not implement balancer.Picker (wrong type for Pick method)
have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
want Pick(balancer.PickInfo) (balancer.PickResult, error)
```
解决方法:
```golang
replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
```

View File

@@ -1,8 +1,13 @@
package command
import (
"fmt"
"os"
"path/filepath"
"github.com/tal-tech/go-zero/tools/goctl/rpc/ctx"
"github.com/tal-tech/go-zero/tools/goctl/rpc/gen"
"github.com/tal-tech/go-zero/tools/goctl/util"
"github.com/urfave/cli"
)
@@ -17,6 +22,39 @@ func RpcTemplate(c *cli.Context) error {
out := c.String("out")
idea := c.Bool("idea")
generator := gen.NewRpcTemplate(out, idea)
generator.MustGenerate()
generator.MustGenerate(true)
return nil
}
func RpcNew(c *cli.Context) error {
idea := c.Bool("idea")
arg := c.Args().First()
if len(arg) == 0 {
arg = "greet"
}
abs, err := filepath.Abs(arg)
if err != nil {
return err
}
_, err = os.Stat(abs)
if err != nil {
if !os.IsNotExist(err) {
return err
}
err = util.MkdirIfNotExist(abs)
if err != nil {
return err
}
}
dir := filepath.Base(filepath.Clean(abs))
protoSrc := filepath.Join(abs, fmt.Sprintf("%v.proto", dir))
templateGenerator := gen.NewRpcTemplate(protoSrc, idea)
templateGenerator.MustGenerate(false)
rpcCtx := ctx.MustCreateRpcContext(protoSrc, "", "", idea)
generator := gen.NewDefaultRpcGenerator(rpcCtx)
rpcCtx.Must(generator.Generate())
return nil
}

View File

@@ -30,13 +30,12 @@ type RpcContext struct {
ProtoFileSrc string
ProtoSource string
TargetDir string
IsInGoEnv bool
console.Console
}
func MustCreateRpcContext(protoSrc, targetDir, serviceName string, idea bool) *RpcContext {
log := console.NewConsole(idea)
info, err := project.Prepare(targetDir, true)
log.Must(err)
if stringx.From(protoSrc).IsEmptyOrSpace() {
log.Fatalln("expected proto source, but nothing found")
@@ -62,6 +61,9 @@ func MustCreateRpcContext(protoSrc, targetDir, serviceName string, idea bool) *R
log.Fatalln("service name is not found")
}
info, err := project.Prepare(targetDir, true)
log.Must(err)
return &RpcContext{
ProjectPath: info.Path,
ProjectName: stringx.From(info.Name),
@@ -71,6 +73,7 @@ func MustCreateRpcContext(protoSrc, targetDir, serviceName string, idea bool) *R
ProtoFileSrc: srcFp,
ProtoSource: filepath.Base(srcFp),
TargetDir: targetDirFp,
IsInGoEnv: info.IsInGoEnv,
Console: log,
}
}

View File

@@ -42,6 +42,11 @@ func (g *defaultRpcGenerator) Generate() (err error) {
return
}
err = g.initGoMod()
if err != nil {
return
}
err = g.genEtc()
if err != nil {
return
@@ -82,5 +87,5 @@ func (g *defaultRpcGenerator) Generate() (err error) {
return
}
return nil
return
}

View File

@@ -113,10 +113,7 @@ func (g *defaultRpcGenerator) genCall() error {
}
service := file.Service[0]
callPath, err := filepath.Abs(service.Name.Lower())
if err != nil {
return err
}
callPath := filepath.Join(g.dirM[dirTarget], service.Name.Lower())
if err = util.MkdirIfNotExist(callPath); err != nil {
return err
@@ -152,7 +149,7 @@ func (g *defaultRpcGenerator) genCall() error {
}
mockFile := filepath.Join(callPath, fmt.Sprintf("%s_mock.go", service.Name.Lower()))
os.Remove(mockFile)
_ = os.Remove(mockFile)
err = util.With("shared").GoFmt(true).Parse(callTemplateText).SaveTo(map[string]interface{}{
"name": service.Name.Lower(),
"head": head,
@@ -167,9 +164,9 @@ func (g *defaultRpcGenerator) genCall() error {
return err
}
// if mockgen is already installed, it will generate code of gomock for shared files
_, err = exec.LookPath("mockgen")
if mockGenInstalled {
execx.Run(fmt.Sprintf("go generate %s", filename), "")
// Deprecated: it will be removed
if mockGenInstalled && g.Ctx.IsInGoEnv {
_, _ = execx.Run(fmt.Sprintf("go generate %s", filename), "")
}
return nil

View File

@@ -0,0 +1,22 @@
package gen
import (
"fmt"
"github.com/tal-tech/go-zero/core/logx"
"github.com/tal-tech/go-zero/tools/goctl/rpc/execx"
)
func (g *defaultRpcGenerator) initGoMod() error {
if !g.Ctx.IsInGoEnv {
projectDir := g.dirM[dirTarget]
cmd := fmt.Sprintf("go mod init %s", g.Ctx.ProjectName.Source())
output, err := execx.Run(fmt.Sprintf(cmd), projectDir)
if err != nil {
logx.Error(err)
return err
}
g.Ctx.Info(output)
}
return nil
}

View File

@@ -1,26 +1,28 @@
package gen
import (
"path/filepath"
"strings"
"github.com/tal-tech/go-zero/tools/goctl/util"
"github.com/tal-tech/go-zero/tools/goctl/util/console"
"github.com/tal-tech/go-zero/tools/goctl/util/stringx"
)
const rpcTemplateText = `syntax = "proto3";
package remote;
package {{.package}};
message Request {
string username = 1;
string password = 2;
string ping = 1;
}
message Response {
string name = 1;
string gender = 2;
string pong = 1;
}
service User {
rpc Login(Request) returns(Response);
service {{.serviceName}} {
rpc Ping(Request) returns(Response);
}
`
@@ -36,8 +38,15 @@ func NewRpcTemplate(out string, idea bool) *rpcTemplate {
}
}
func (r *rpcTemplate) MustGenerate() {
err := util.With("t").Parse(rpcTemplateText).SaveTo(nil, r.out, false)
func (r *rpcTemplate) MustGenerate(showState bool) {
protoFilename := filepath.Base(r.out)
serviceName := stringx.From(strings.TrimSuffix(protoFilename, filepath.Ext(protoFilename)))
err := util.With("t").Parse(rpcTemplateText).SaveTo(map[string]string{
"package": serviceName.UnTitle(),
"serviceName": serviceName.Title(),
}, r.out, false)
r.Must(err)
r.Success("Done.")
if showState {
r.Success("Done.")
}
}

View File

@@ -2,6 +2,7 @@ package project
import (
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
@@ -23,7 +24,9 @@ type (
Path string // Project path name
Name string // Project name
Package string // The service related package
GoMod GoMod
// true-> project in go path or project init with go mod,or else->false
IsInGoEnv bool
GoMod GoMod
}
GoMod struct {
@@ -61,7 +64,11 @@ func Prepare(projectDir string, checkGrpcEnv bool) (*Project, error) {
if err != nil {
return nil, err
}
goMod = strings.TrimSpace(ret)
if goMod == os.DevNull {
goMod = ""
}
ret, err = execx.Run(constGoPath, "")
if err != nil {
@@ -70,6 +77,7 @@ func Prepare(projectDir string, checkGrpcEnv bool) (*Project, error) {
goPath = strings.TrimSpace(ret)
src := filepath.Join(goPath, "src")
var isInGoEnv = true
if len(goMod) > 0 {
path = filepath.Dir(goMod)
name = filepath.Base(path)
@@ -98,6 +106,7 @@ func Prepare(projectDir string, checkGrpcEnv bool) (*Project, error) {
name = filepath.Clean(filepath.Base(absPath))
path = projectDir
pkg = name
isInGoEnv = false
} else {
r := strings.TrimPrefix(pwd, src+string(filepath.Separator))
name = filepath.Dir(r)
@@ -111,9 +120,10 @@ func Prepare(projectDir string, checkGrpcEnv bool) (*Project, error) {
}
return &Project{
Name: name,
Path: path,
Package: pkg,
Name: name,
Path: path,
Package: pkg,
IsInGoEnv: isInGoEnv,
GoMod: GoMod{
Module: module,
Path: goMod,