refine goctl rpc generator

doc/shorturl.md

@@ -102,7 +102,7 @@
 * 测试API Gateway服务
 
   ```shell
-  curl -i "http://localhost:8888/shorten?url=a"
+  curl -i "http://localhost:8888/shorten?url=http://www.xiaoheiban.cn"
   ```
 
   返回如下：
@@ -116,6 +116,8 @@
   {"shortUrl":""}
   ```
 
+  可以看到我们API Gateway其实啥也没干，就返回了个空值，接下来我们会在rpc服务里实现业务逻辑
+
 * 可以修改`internal/svc/servicecontext.go`来传递服务依赖（如果需要）
 
 * 实现逻辑可以修改`internal/logic`下的对应文件
@@ -125,11 +127,65 @@
 ## 4. 编写shorten rpc服务（未完）
 
 * 编写`shorten.proto`文件
+
+  可以通过命令生成proto文件模板
+
+  ```shell
+  goctl rpc template -o shorten.proto
+  ```
+
+  修改后文件内容如下：
+
+  ```protobuf
+  syntax = "proto3";
+  
+  package shorten;
+  
+  message shortenReq {
+      string url = 1;
+  }
+  
+  message shortenResp {
+      string key = 1;
+  }
+  
+  service shortener {
+      rpc shorten(shortenReq) returns(shortenResp);
+  }
+  ```
+
 * 用`goctl`生成rpc代码
 
 ## 5. 编写expand rpc服务（未完）
 
 * 编写`expand.proto`文件
+
+  可以通过命令生成proto文件模板
+
+  ```shell
+  goctl rpc template -o expand.proto
+  ```
+
+  修改后文件内容如下：
+
+  ```protobuf
+  syntax = "proto3";
+  
+  package expand;
+  
+  message expandReq {
+      string key = 1;
+  }
+  
+  message expandResp {
+      string url = 1;
+  }
+  
+  service expander {
+      rpc expand(expandReq) returns(expandResp);
+  }
+  ```
+
 * 用`goctl`生成rpc代码
 
 ## 6. 修改API Gateway代码调用shorten/expand rpc服务（未完）
@@ -150,6 +206,16 @@
   ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
   ```
 
+* 创建DB和table
+
+  ```sql
+  create database gozero;
+  ```
+
+  ```sql
+  source shorturl.sql;
+  ```
+
 * 在`rpc/model`目录下执行如下命令生成CRUD+cache代码，`-c`表示使用`redis cache`
 
   ```shell
@@ -197,4 +263,10 @@
 
 ## 10. Benchmark（未完）
 
-## 11. 总结（未完）
+## 11. 总结（未完）
+
+可以看到go-zero不只是一个框架，更是一个建立在框架+工具基础上的，简化和规范了整个微服务构建的技术体系。
+
+我们一直强调**工具大于约定和文档**。
+
+另外，我们在保持简单的同时也尽可能把微服务治理的复杂度封装到了框架内部，极大的降低了开发人员的心智负担，使得业务开发得以快速推进。

tools/goctl/goctl.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/urfave/cli"
-
 	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/tools/goctl/api/apigen"
 	"github.com/tal-tech/go-zero/tools/goctl/api/dartgen"
@@ -21,6 +19,7 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/feature"
 	model "github.com/tal-tech/go-zero/tools/goctl/model/sql/command"
 	rpc "github.com/tal-tech/go-zero/tools/goctl/rpc/command"
+	"github.com/urfave/cli"
 )
 
 var (
@@ -196,7 +195,7 @@ var (
 			Subcommands: []cli.Command{
 				{
 					Name:  "template",
-					Usage: `generate proto template"`,
+					Usage: `generate proto template`,
 					Flags: []cli.Flag{
 						cli.StringFlag{
 							Name:  "out, o",
@@ -211,7 +210,7 @@ var (
 				},
 				{
 					Name:  "proto",
-					Usage: `generate rpc from proto"`,
+					Usage: `generate rpc from proto`,
 					Flags: []cli.Flag{
 						cli.StringFlag{
 							Name:  "src, s",
@@ -227,7 +226,7 @@ var (
 						},
 						cli.StringFlag{
 							Name:  "shared",
-							Usage: `the dir of the shared file,default path is "${pwd}/shared. [option]"`,
+							Usage: `the dir of the shared file,default path is "${pwd}/shared. [option]`,
 						},
 						cli.BoolFlag{
 							Name:  "idea",

tools/goctl/rpc/README.md

@@ -17,7 +17,7 @@ $ goctl rpc template -o=user.proto
 ```golang
 syntax = "proto3";
 
-package remoteuser;
+package remote;
 
 message Request {
   // 用户名
@@ -33,7 +33,7 @@ message Response {
   string gender = 2;
 }
 
-service User{
+service User {
   // 登录
   rpc Login(Request)returns(Response);
 }
@@ -82,17 +82,17 @@ user
 
 ### go mod 工程  
   对于`$ go version`不低于1.5版本的的工程，会优先寻找`$ go env GOMODCACHE`目录，如果没有则去`${GOPATH}`中查找（见下文）,而低于1.5版本的则会优先从`${GOPATH}/pkg/mod`目录下查找，否则也从`% GOPATH`中查找（见下文）
-  
+
 ### go path工程
   对于没有使用go mod的工程，则默认当作在`${GOPATH}`中处理（暂不支持用户自定义的GOPATH），而这种情况下则会默认从`${GOPATH}/src`中查找
-  
+
 > 注意： 
  * 对于以上两种工程如果没有在对应目录查找到`protoc-gen-go`则会提示相应错误，尽管`protoc-gen-go`可能在其他已经设置环境变量的目录中，这个将在后面版本进行优化。
  * 对于go mod工程，如果工程没有依赖`github.com/golang/protobuf`则需要提前引入。
- 
+
 ### 好处
 * 保证grpc代码生成规范的一致性
- 
+
 # 用法
 ```shell script
 $ goctl rpc proto -h
@@ -126,11 +126,11 @@ OPTIONS:
         ```
         则服务名称亦为user，而非proto所在文件夹名称了，这里推荐使用这种结构，可以方便在同一个服务名下建立不同类型的服务(api、rpc、mq等)，便于代码管理与维护。
     * --shared 非必填，默认为$dir(xxx.proto)/shared，rpc client逻辑代码存放目录。
-        
+      
       > 注意：这里的shared文件夹名称将会是代码中的package名称。
     
     * --idea 非必填，是否为idea插件中执行，保留字段，终端执行可以忽略
-     
+    
 # 开发人员需要做什么
 
 关注业务代码编写，将重复性、与业务无关的工作交给goctl，生成好rpc服务代码后，开饭人员仅需要修改
@@ -146,7 +146,7 @@ OPTIONS:
 * proto不支持外部依赖包引入，message不支持inline
 * 目前main文件、shared文件、handler文件会被强制覆盖，而和开发人员手动需要编写的则不会覆盖生成，这一类在代码头部均有
     ```shell script
-    // Code generated by goctl. DO NOT EDIT.
+    // Code generated by goctl. DO NOT EDIT!
     // Source: xxx.proto
     ```
   的标识，请注意不要将也写业务性代码写在里面。

tools/goctl/rpc/command/command.go

@@ -4,7 +4,7 @@ import (
 	"github.com/urfave/cli"
 
 	"github.com/tal-tech/go-zero/tools/goctl/rpc/ctx"
-	"github.com/tal-tech/go-zero/tools/goctl/rpc/goen"
+	"github.com/tal-tech/go-zero/tools/goctl/rpc/gen"
 )
 
 func Rpc(c *cli.Context) error {

tools/goctl/rpc/ctx/project.go

@@ -16,9 +16,7 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util/console"
 )
 
-var (
+var errProtobufNotFound = errors.New("github.com/golang/protobuf is not found,please ensure you has already [go get github.com/golang/protobuf]")
-	errProtobufNotFound = errors.New("github.com/golang/protobuf is not found,please ensure you has already [go get github.com/golang/protobuf]")
-)
 
 const (
 	constGo          = "go"
@@ -51,7 +49,7 @@ type (
 )
 
 func prepare(log console.Console) (*Project, error) {
-	log.Info("check go env ...")
+	log.Info("checking go env...")
 	_, err := exec.LookPath(constGo)
 	if err != nil {
 		return nil, err
@@ -109,11 +107,6 @@ func prepare(log console.Console) (*Project, error) {
 		if err != nil {
 			return nil, err
 		}
-
-		protobufModule, err = matchProtoBuf(data)
-		if err != nil {
-			return nil, err
-		}
 	} else {
 		if goModCache == "" {
 			goModCache = src
@@ -180,21 +173,6 @@ func prepare(log console.Console) (*Project, error) {
 	}, nil
 }
 
-// github.com/golang/protobuf@{version}
-func matchProtoBuf(data []byte) (string, error) {
-	text := string(data)
-	re := regexp.MustCompile(`(?m)(github.com/golang/protobuf)\s+(v[0-9.]+)`)
-	matches := re.FindAllStringSubmatch(text, -1)
-	if len(matches) == 0 {
-		return "", errProtobufNotFound
-	}
-	groups := matches[0]
-	if len(groups) < 3 {
-		return "", errProtobufNotFound
-	}
-	return fmt.Sprintf("%s@%s", groups[1], groups[2]), nil
-}
-
 func matchModule(data []byte) (string, error) {
 	text := string(data)
 	re := regexp.MustCompile(`(?m)^\s*module\s+[a-z0-9/\-.]+$`)
@@ -204,5 +182,6 @@ func matchModule(data []byte) (string, error) {
 		index := strings.Index(target, "module")
 		return strings.TrimSpace(target[index+6:]), nil
 	}
+
 	return "", nil
 }

tools/goctl/rpc/gen/gen.go

@@ -34,7 +34,7 @@ func NewDefaultRpcGenerator(ctx *ctx.RpcContext) *defaultRpcGenerator {
 }
 
 func (g *defaultRpcGenerator) Generate() (err error) {
-	g.Ctx.Info("code generating...")
+	g.Ctx.Info("generating code...")
 	defer func() {
 		if err == nil {
 			g.Ctx.Success("Done.")

tools/goctl/rpc/gen/genconfig.go

@@ -8,15 +8,13 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )
 
-var configTemplate = `package config
+const configTemplate = `package config
 
 import "github.com/tal-tech/go-zero/rpcx"
 
-type (
+type Config struct {
-	Config struct {
+	rpcx.RpcServerConf
-		rpcx.RpcServerConf
+}
-	}
-)
 `
 
 func (g *defaultRpcGenerator) genConfig() error {

tools/goctl/rpc/gen/genetc.go

@@ -0,0 +1,30 @@
+package gogen
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/tal-tech/go-zero/tools/goctl/util"
+)
+
+const etcTemplate = `Name: {{.serviceName}}.rpc
+Log:
+  Mode: console
+ListenOn: 127.0.0.1:8080
+Etcd:
+  Hosts:
+  - 127.0.0.1:6379
+  Key: {{.serviceName}}.rpc
+`
+
+func (g *defaultRpcGenerator) genEtc() error {
+	etdDir := g.dirM[dirEtc]
+	fileName := filepath.Join(etdDir, fmt.Sprintf("%v.yaml", g.Ctx.ServiceName.Lower()))
+	if util.FileExists(fileName) {
+		return nil
+	}
+
+	return util.With("etc").Parse(etcTemplate).SaveTo(map[string]interface{}{
+		"serviceName": g.Ctx.ServiceName.Lower(),
+	}, fileName, false)
+}

tools/goctl/rpc/gen/genhandler.go

@@ -8,18 +8,14 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )
 
-var (
+const (
 	remoteTemplate = `{{.head}}
 
 package handler
 
-import (
+import {{.imports}}
-	{{.imports}}
-)
 
-type (
+type {{.types}}
-	{{.types}}
-)
 
 {{.newFuncs}}
 `
@@ -33,9 +29,11 @@ import (
 	{{.imports}}
 )
 
+type {{.server}}Server struct{}
+
 {{if .hasComment}}{{.comment}}{{end}}
 func (s *{{.server}}Server) {{.method}} (ctx context.Context, in *{{.package}}.{{.request}}) (*{{.package}}.{{.response}}, error) {
-	l:=logic.New{{.logicName}}(ctx,s.svcCtx)
+	l := logic.New{{.logicName}}(ctx,s.svcCtx)
 	return l.{{.method}}(in)
 }
 `

tools/goctl/rpc/gen/genlogic.go

@@ -10,29 +10,26 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )
 
-var (
+const (
 	logicTemplate = `package logic
 
 import (
 	"context"
 
 	{{.imports}}
+
 	"github.com/tal-tech/go-zero/core/logx"
 )
 
-type (
+type {{.logicName}} struct {
-	{{.logicName}} struct {
+	ctx context.Context
-		ctx context.Context
+	logx.Logger
-		logx.Logger
+}
-		// todo: add your logic here and delete this line
-	}
-)
 
 func New{{.logicName}}(ctx context.Context,svcCtx *svc.ServiceContext) *{{.logicName}} {
 	return &{{.logicName}}{
 		ctx:    ctx,
 		Logger: logx.WithContext(ctx),
-		// todo: add your logic here and delete this line
 	}
 }
 {{.functions}}
@@ -40,6 +37,7 @@ func New{{.logicName}}(ctx context.Context,svcCtx *svc.ServiceContext) *{{.logic
 	logicFunctionTemplate = `{{if .hasComment}}{{.comment}}{{end}}
 func (l *{{.logicName}}) {{.method}} (in *{{.package}}.{{.request}}) (*{{.package}}.{{.response}}, error) {
 	var resp {{.package}}.{{.response}}
+
 	// todo: add your logic here and delete this line
 	
 	return &resp,nil

tools/goctl/rpc/gen/genmain.go

@@ -9,7 +9,7 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )
 
-var mainTemplate = `{{.head}}
+const mainTemplate = `{{.head}}
 
 package main
 
@@ -18,15 +18,14 @@ import (
 	"fmt"
 	"log"
 
-	"google.golang.org/grpc"
+	{{.imports}}
 
 	"github.com/tal-tech/go-zero/core/conf"
 	"github.com/tal-tech/go-zero/rpcx"
-
+	"google.golang.org/grpc"
-	{{.imports}}
 )
 
-var configFile = flag.String("f", "etc/{{.serviceName}}.json", "the config file")
+var configFile = flag.String("f", "etc/{{.serviceName}}.yaml", "the config file")
 
 func main() {
 	flag.Parse()
@@ -42,10 +41,10 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
 	fmt.Printf("Starting rpc server at %s...\n", c.ListenOn)
 	s.Start()
 }
-
 `
 
 func (g *defaultRpcGenerator) genMain() error {

tools/goctl/rpc/gen/genpb.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"github.com/dsymonds/gotoc/parser"
-
 	"github.com/tal-tech/go-zero/core/lang"
 	"github.com/tal-tech/go-zero/tools/goctl/rpc/execx"
 	astParser "github.com/tal-tech/go-zero/tools/goctl/rpc/parser"
@@ -80,6 +79,9 @@ protoc -I=%s --go_out=plugins=grpc:%s %s`, filepath.Join(g.Ctx.GoPath, "bin"), s
 		return err
 	}
 
-	g.Ctx.Info(stdout)
+	if len(stdout) > 0 {
+		g.Ctx.Info(stdout)
+	}
+
 	return nil
 }

tools/goctl/rpc/gen/genshared.go

@@ -12,10 +12,10 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )
 
-var (
+const (
 	sharedTemplateText = `{{.head}}
 
-//go:generate mockgen -destination ./mock{{.name}}model.go -package {{.filePackage}} -source $GOFILE
+//go:generate mockgen -destination ./{{.name}}model_mock.go -package {{.filePackage}} -source $GOFILE
 
 package {{.filePackage}}
 
@@ -23,6 +23,7 @@ import (
 	"context"
 
 	{{.package}}
+
 	"github.com/tal-tech/go-zero/core/jsonx"
 	"github.com/tal-tech/go-zero/rpcx"
 )
@@ -31,13 +32,13 @@ type (
 	{{.serviceName}}Model interface {
 		{{.interface}}
 	}
+
 	default{{.serviceName}}Model struct {
 		cli rpcx.Client
 	}
 )
 
-
+func New{{.serviceName}}Model(cli rpcx.Client) {{.serviceName}}Model {
-func NewDefault{{.serviceName}}Model(cli rpcx.Client) {{.serviceName}}Model {
 	return &default{{.serviceName}}Model{
 		cli: cli,
 	}
@@ -49,51 +50,52 @@ func NewDefault{{.serviceName}}Model(cli rpcx.Client) {{.serviceName}}Model {
 
 package {{.filePackage}}
 
-import (
+import "errors"
-	"errors"
-)
 
-var (
+var errJsonConvert = errors.New("json convert error")
-	errJsonConvert = errors.New("json convert error")
-)
 
 {{.types}}
-
 `
 	sharedInterfaceFunctionTemplate = `{{if .hasComment}}{{.comment}}
 {{end}}{{.method}}(ctx context.Context,in *{{.pbRequest}}) {{if .hasResponse}}(*{{.pbResponse}},{{end}} error{{if .hasResponse}}){{end}}`
 	sharedFunctionTemplate = `
 {{if .hasComment}}{{.comment}}{{end}}
 func (m *default{{.rpcServiceName}}Model) {{.method}}(ctx context.Context,in *{{.pbRequest}}) {{if .hasResponse}}(*{{.pbResponse}},{{end}} error{{if .hasResponse}}){{end}} {
-	conn:= m.cli.Conn()
+	client := {{.package}}.New{{.rpcServiceName}}Client(m.cli.Conn())
-	client := {{.package}}.New{{.rpcServiceName}}Client(conn)
 	var request {{.package}}.{{.pbRequest}}
 	bts, err := jsonx.Marshal(in)
 	if err != nil {
-		return {{if .hasResponse}}nil,{{end}}errJsonConvert
+		return {{if .hasResponse}}nil, {{end}}errJsonConvert
 	}
+
 	err = jsonx.Unmarshal(bts, &request)
 	if err != nil {
-		return {{if .hasResponse}}nil,{{end}}errJsonConvert
+		return {{if .hasResponse}}nil, {{end}}errJsonConvert
 	}
-	{{if .hasResponse}}resp,err:={{else}}_,err={{end}}client.{{.method}}(ctx, &request)
+
-	{{if .hasResponse}}if err!=nil{
+	{{if .hasResponse}}resp, err := {{else}}_, err = {{end}}client.{{.method}}(ctx, &request)
-		return nil,err
+	{{if .hasResponse}}if err != nil{
+		return nil, err
 	}
+
 	var ret {{.pbResponse}}
-	bts,err=jsonx.Marshal(resp)
+	bts, err = jsonx.Marshal(resp)
-	if err!=nil{
+	if err != nil{
-		return nil,errJsonConvert
+		return nil, errJsonConvert
 	}
-	err=jsonx.Unmarshal(bts,&ret)
+
-	if err!=nil{
+	err = jsonx.Unmarshal(bts, &ret)
-		return nil,errJsonConvert
+	if err != nil{
+		return nil, errJsonConvert
 	}
-	return &ret, nil{{else}}if err!=nil {
+
+	return &ret, nil{{else}}if err != nil {
 		return err
 	}
+
 	return nil{{end}}
-}`
+}
+`
 )
 
 func (g *defaultRpcGenerator) genShared() error {
@@ -127,7 +129,7 @@ func (g *defaultRpcGenerator) genShared() error {
 		if err != nil {
 			return err
 		}
-		mockFile := filepath.Join(g.Ctx.SharedDir, fmt.Sprintf("mock%smodel.go", service.Name.Lower()))
+		mockFile := filepath.Join(g.Ctx.SharedDir, fmt.Sprintf("%smodel_mock.go", service.Name.Lower()))
 		os.Remove(mockFile)
 		err = util.With("shared").GoFmt(true).Parse(sharedTemplateText).SaveTo(map[string]interface{}{
 			"name":        service.Name.Lower(),

tools/goctl/rpc/gen/gensvc.go

@@ -7,16 +7,13 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )
 
-var svcTemplate = `package svc
+const svcTemplate = `package svc
 
 import {{.imports}}
 
-type (
+type ServiceContext struct {
-	ServiceContext struct {
+	c config.Config
-		c config.Config
+}
-		// todo: add your logic here and delete this line
-	}
-)
 
 func NewServiceContext(c config.Config) *ServiceContext {
 	return &ServiceContext{

tools/goctl/rpc/gen/template.go

@@ -5,9 +5,9 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util/console"
 )
 
-var rpcTemplateText = `syntax = "proto3";
+const rpcTemplateText = `syntax = "proto3";
 
-package remoteuser;
+package remote;
 
 message Request {
   string username = 1;
@@ -19,16 +19,15 @@ message Response {
   string gender = 2;
 }
 
-service User{
+service User {
-  rpc Login(Request)returns(Response);
+  rpc Login(Request) returns(Response);
-}`
+}
+`
 
-type (
+type rpcTemplate struct {
-	rpcTemplate struct {
+	out string
-		out string
+	console.Console
-		console.Console
+}
-	}
-)
 
 func NewRpcTemplate(out string, idea bool) *rpcTemplate {
 	return &rpcTemplate{

tools/goctl/rpc/goen/genetc.go

@@ -1,34 +0,0 @@
-package gogen
-
-import (
-	"fmt"
-	"path/filepath"
-
-	"github.com/tal-tech/go-zero/tools/goctl/util"
-)
-
-var etcTemplate = `{
-  "Name": "{{.serviceName}}.rpc",
-  "Log": {
-    "Mode": "console"
-  },
-  "ListenOn": "127.0.0.1:8080",
-  "Etcd": {
-    "Hosts": ["127.0.0.1:6379"],
-    "Key": "{{.serviceName}}.rpc"
-  }
-}
-`
-
-func (g *defaultRpcGenerator) genEtc() error {
-	etdDir := g.dirM[dirEtc]
-	fileName := filepath.Join(etdDir, fmt.Sprintf("%v.json", g.Ctx.ServiceName.Lower()))
-	if util.FileExists(fileName) {
-		return nil
-	}
-	return util.With("etc").
-		Parse(etcTemplate).
-		SaveTo(map[string]interface{}{
-			"serviceName": g.Ctx.ServiceName.Lower(),
-		}, fileName, false)
-}

tools/goctl/rpc/parser/pbast.go

@@ -435,7 +435,7 @@ func (a *PbAst) GenTypesCode() (string, error) {
 		types = append(types, structCode)
 	}
 	buffer, err := util.With("type").Parse(typeTemplate).Execute(map[string]interface{}{
-		"types": strings.Join(types, "\n"),
+		"types": strings.Join(types, "\n\n"),
 	})
 	if err != nil {
 		return "", err

tools/goctl/util/head.go

@@ -1,6 +1,6 @@
 package util
 
-var headTemplate = `// Code generated by goctl. DO NOT EDIT.
+var headTemplate = `// Code generated by goctl. DO NOT EDIT!
 // Source: {{.source}}`
 
 func GetHead(source string) string {