feature: refactor api parse to g4 (#365)

* feature: refactor api parse to g4

* new g4 parser

* add CHANGE_LOG.MD

* refactor

* fix byte bug

* refactor

* optimized

* optimized

* revert

* update readme.md

* update readme.md

* update readme.md

* update readme.md

* remove no need

* fix java gen

* add upgrade

* resolve confilits

Co-authored-by: anqiansong <anqiansong@xiaoheiban.cn>
This commit is contained in:
kingxt
2021-01-11 15:10:51 +08:00
committed by GitHub
parent b0ccfb8eb4
commit ee19fb736b
88 changed files with 13641 additions and 2458 deletions

View File

@@ -41,11 +41,7 @@ func GoCommand(c *cli.Context) error {
}
func DoGenProject(apiFile, dir, style string) error {
p, err := parser.NewParser(apiFile)
if err != nil {
return err
}
api, err := p.Parse()
api, err := parser.Parse(apiFile)
if err != nil {
return err
}

View File

@@ -16,9 +16,9 @@ import (
const testApiTemplate = `
info(
title: doc title
desc: >
desc: ">
doc description first part,
doc description second part<
doc description second part<"
version: 1.0
)
@@ -55,9 +55,7 @@ service A-api {
const testMultiServiceTemplate = `
info(
title: doc title
desc: >
doc description first part,
doc description second part<
desc: doc description first part
version: 1.0
)
@@ -229,7 +227,7 @@ type Response struct {
}
service A-api {
@doc(helloworld)
@doc ("helloworld")
@server(
handler: GreetHandler
)
@@ -249,7 +247,7 @@ type Response struct {
}
service A-api {
@doc(helloworld)
@doc ("helloworld")
@server(
handler: GreetHandler
)
@@ -325,10 +323,7 @@ func TestParser(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
api, err := parser.Parse()
api, err := parser.Parse(filename)
assert.Nil(t, err)
assert.Equal(t, len(api.Types), 2)
@@ -337,8 +332,8 @@ func TestParser(t *testing.T) {
assert.Equal(t, api.Service.Routes()[0].Path, "/greet/from/:name")
assert.Equal(t, api.Service.Routes()[1].Path, "/greet/get")
assert.Equal(t, api.Service.Routes()[1].RequestType.Name, "Request")
assert.Equal(t, api.Service.Routes()[1].ResponseType.Name, "")
assert.Equal(t, api.Service.Routes()[1].RequestTypeName(), "Request")
assert.Equal(t, api.Service.Routes()[1].ResponseType, nil)
validate(t, filename)
}
@@ -349,10 +344,7 @@ func TestMultiService(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
api, err := parser.Parse()
api, err := parser.Parse(filename)
assert.Nil(t, err)
assert.Equal(t, len(api.Service.Routes()), 2)
@@ -367,10 +359,7 @@ func TestApiNoInfo(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
_, err = parser.Parse()
_, err = parser.Parse(filename)
assert.Nil(t, err)
validate(t, filename)
@@ -382,7 +371,7 @@ func TestInvalidApiFile(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
_, err = parser.NewParser(filename)
_, err = parser.Parse(filename)
assert.NotNil(t, err)
}
@@ -392,14 +381,11 @@ func TestAnonymousAnnotation(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
api, err := parser.Parse()
api, err := parser.Parse(filename)
assert.Nil(t, err)
assert.Equal(t, len(api.Service.Routes()), 1)
assert.Equal(t, api.Service.Routes()[0].Annotations[0].Value, "GreetHandler")
assert.Equal(t, api.Service.Routes()[0].Handler, "GreetHandler")
validate(t, filename)
}
@@ -410,10 +396,7 @@ func TestApiHasMiddleware(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
_, err = parser.Parse()
_, err = parser.Parse(filename)
assert.Nil(t, err)
validate(t, filename)
@@ -425,10 +408,7 @@ func TestApiHasJwt(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
_, err = parser.Parse()
_, err = parser.Parse(filename)
assert.Nil(t, err)
validate(t, filename)
@@ -440,10 +420,7 @@ func TestApiHasJwtAndMiddleware(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
_, err = parser.Parse()
_, err = parser.Parse(filename)
assert.Nil(t, err)
validate(t, filename)
@@ -455,13 +432,8 @@ func TestApiHasNoRequestBody(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
_, err = parser.Parse(filename)
assert.Nil(t, err)
_, err = parser.Parse()
assert.Nil(t, err)
validate(t, filename)
}
func TestApiRoutes(t *testing.T) {
@@ -470,10 +442,7 @@ func TestApiRoutes(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
_, err = parser.Parse()
_, err = parser.Parse(filename)
assert.Nil(t, err)
validate(t, filename)
@@ -485,10 +454,7 @@ func TestHasCommentRoutes(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
_, err = parser.Parse()
_, err = parser.Parse(filename)
assert.Nil(t, err)
validate(t, filename)
@@ -500,13 +466,8 @@ func TestInlineTypeNotExist(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
_, err = parser.Parse()
assert.Nil(t, err)
validate(t, filename)
_, err = parser.Parse(filename)
assert.NotNil(t, err)
}
func TestHasImportApi(t *testing.T) {
@@ -520,15 +481,12 @@ func TestHasImportApi(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(importApiName)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
api, err := parser.Parse()
api, err := parser.Parse(filename)
assert.Nil(t, err)
var hasInline bool
for _, ty := range api.Types {
if ty.Name == "ImportData" {
if ty.Name() == "ImportData" {
hasInline = true
break
}
@@ -544,10 +502,7 @@ func TestNoStructApi(t *testing.T) {
assert.Nil(t, err)
defer os.Remove(filename)
parser, err := parser.NewParser(filename)
assert.Nil(t, err)
spec, err := parser.Parse()
spec, err := parser.Parse(filename)
assert.Nil(t, err)
assert.Equal(t, len(spec.Types), 5)
@@ -559,8 +514,8 @@ func TestNestTypeApi(t *testing.T) {
err := ioutil.WriteFile(filename, []byte(nestTypeApi), os.ModePerm)
assert.Nil(t, err)
defer os.Remove(filename)
_, err = parser.NewParser(filename)
_, err = parser.Parse(filename)
assert.NotNil(t, err)
}
@@ -569,7 +524,8 @@ func TestCamelStyle(t *testing.T) {
err := ioutil.WriteFile(filename, []byte(testApiTemplate), os.ModePerm)
assert.Nil(t, err)
defer os.Remove(filename)
_, err = parser.NewParser(filename)
_, err = parser.Parse(filename)
assert.Nil(t, err)
validateWithCamel(t, filename, "GoZero")

View File

@@ -5,7 +5,6 @@ import (
"strconv"
"github.com/tal-tech/go-zero/tools/goctl/api/spec"
"github.com/tal-tech/go-zero/tools/goctl/api/util"
"github.com/tal-tech/go-zero/tools/goctl/config"
"github.com/tal-tech/go-zero/tools/goctl/util/format"
)
@@ -26,14 +25,8 @@ func genEtc(dir string, cfg *config.Config, api *spec.ApiSpec) error {
}
service := api.Service
host, ok := util.GetAnnotationValue(service.Groups[0].Annotations, "server", "host")
if !ok {
host = "0.0.0.0"
}
port, ok := util.GetAnnotationValue(service.Groups[0].Annotations, "server", "port")
if !ok {
port = strconv.Itoa(defaultPort)
}
host := "0.0.0.0"
port := strconv.Itoa(defaultPort)
return genFile(fileGenConfig{
dir: dir,

View File

@@ -1,14 +1,11 @@
package gogen
import (
"errors"
"fmt"
"path"
"strings"
"unicode"
"github.com/tal-tech/go-zero/tools/goctl/api/spec"
apiutil "github.com/tal-tech/go-zero/tools/goctl/api/util"
"github.com/tal-tech/go-zero/tools/goctl/config"
"github.com/tal-tech/go-zero/tools/goctl/util"
"github.com/tal-tech/go-zero/tools/goctl/util/format"
@@ -65,11 +62,11 @@ func genHandler(dir string, cfg *config.Config, group spec.Group, route spec.Rou
return doGenToFile(dir, handler, cfg, group, route, Handler{
ImportPackages: genHandlerImports(group, route, parentPkg),
HandlerName: handler,
RequestType: util.Title(route.RequestType.Name),
RequestType: util.Title(route.RequestTypeName()),
LogicType: strings.Title(getLogicName(route)),
Call: strings.Title(strings.TrimSuffix(handler, "Handler")),
HasResp: len(route.ResponseType.Name) > 0,
HasRequest: len(route.RequestType.Name) > 0,
HasResp: len(route.ResponseTypeName()) > 0,
HasRequest: len(route.RequestTypeName()) > 0,
})
}
@@ -109,7 +106,7 @@ func genHandlerImports(group spec.Group, route spec.Route, parentPkg string) str
imports = append(imports, fmt.Sprintf("\"%s\"",
util.JoinPackages(parentPkg, getLogicFolderPath(group, route))))
imports = append(imports, fmt.Sprintf("\"%s\"", util.JoinPackages(parentPkg, contextDir)))
if len(route.RequestType.Name) > 0 {
if len(route.RequestTypeName()) > 0 {
imports = append(imports, fmt.Sprintf("\"%s\"\n", util.JoinPackages(parentPkg, typesDir)))
}
imports = append(imports, fmt.Sprintf("\"%s/rest/httpx\"", vars.ProjectOpenSourceUrl))
@@ -118,18 +115,7 @@ func genHandlerImports(group spec.Group, route spec.Route, parentPkg string) str
}
func getHandlerBaseName(route spec.Route) (string, error) {
handler, ok := apiutil.GetAnnotationValue(route.Annotations, "server", "handler")
if !ok {
return "", fmt.Errorf("missing handler annotation for %q", route.Path)
}
for _, char := range handler {
if !unicode.IsDigit(char) && !unicode.IsLetter(char) {
return "", errors.New(fmt.Sprintf("route [%s] handler [%s] invalid, handler name should only contains letter or digit",
route.Path, handler))
}
}
handler := route.Handler
handler = strings.TrimSpace(handler)
handler = strings.TrimSuffix(handler, "handler")
handler = strings.TrimSuffix(handler, "Handler")
@@ -137,10 +123,10 @@ func getHandlerBaseName(route spec.Route) (string, error) {
}
func getHandlerFolderPath(group spec.Group, route spec.Route) string {
folder, ok := apiutil.GetAnnotationValue(route.Annotations, "server", groupProperty)
if !ok {
folder, ok = apiutil.GetAnnotationValue(group.Annotations, "server", groupProperty)
if !ok {
folder := route.GetAnnotation(groupProperty)
if len(folder) == 0 {
folder = group.GetAnnotation(groupProperty)
if len(folder) == 0 {
return handlerDir
}
}

View File

@@ -6,7 +6,6 @@ import (
"strings"
"github.com/tal-tech/go-zero/tools/goctl/api/spec"
"github.com/tal-tech/go-zero/tools/goctl/api/util"
"github.com/tal-tech/go-zero/tools/goctl/config"
ctlutil "github.com/tal-tech/go-zero/tools/goctl/util"
"github.com/tal-tech/go-zero/tools/goctl/util/format"
@@ -68,16 +67,20 @@ func genLogicByRoute(dir string, cfg *config.Config, group spec.Group, route spe
var responseString string
var returnString string
var requestString string
if len(route.ResponseType.Name) > 0 {
resp := strings.Title(route.ResponseType.Name)
responseString = "(*types." + resp + ", error)"
returnString = fmt.Sprintf("return &types.%s{}, nil", resp)
if len(route.ResponseTypeName()) > 0 {
resp := responseGoTypeName(route, typesPacket)
responseString = "(" + resp + ", error)"
if strings.HasPrefix(resp, "*") {
returnString = fmt.Sprintf("return &%s{}, nil", strings.TrimPrefix(resp, "*"))
} else {
returnString = fmt.Sprintf("return %s{}, nil", resp)
}
} else {
responseString = "error"
returnString = "return nil"
}
if len(route.RequestType.Name) > 0 {
requestString = "req " + "types." + strings.Title(route.RequestType.Name)
if len(route.RequestTypeName()) > 0 {
requestString = "req " + requestGoTypeName(route, typesPacket)
}
return genFile(fileGenConfig{
@@ -100,10 +103,10 @@ func genLogicByRoute(dir string, cfg *config.Config, group spec.Group, route spe
}
func getLogicFolderPath(group spec.Group, route spec.Route) string {
folder, ok := util.GetAnnotationValue(route.Annotations, "server", groupProperty)
if !ok {
folder, ok = util.GetAnnotationValue(group.Annotations, "server", groupProperty)
if !ok {
folder := route.GetAnnotation(groupProperty)
if len(folder) == 0 {
folder = group.GetAnnotation(groupProperty)
if len(folder) == 0 {
return logicDir
}
}
@@ -116,7 +119,7 @@ func genLogicImports(route spec.Route, parentPkg string) string {
var imports []string
imports = append(imports, `"context"`+"\n")
imports = append(imports, fmt.Sprintf("\"%s\"", ctlutil.JoinPackages(parentPkg, contextDir)))
if len(route.ResponseType.Name) > 0 || len(route.RequestType.Name) > 0 {
if len(route.ResponseTypeName()) > 0 || len(route.RequestTypeName()) > 0 {
imports = append(imports, fmt.Sprintf("\"%s\"\n", ctlutil.JoinPackages(parentPkg, typesDir)))
}
imports = append(imports, fmt.Sprintf("\"%s/core/logx\"", vars.ProjectOpenSourceUrl))

View File

@@ -10,7 +10,6 @@ import (
"github.com/tal-tech/go-zero/core/collection"
"github.com/tal-tech/go-zero/tools/goctl/api/spec"
apiutil "github.com/tal-tech/go-zero/tools/goctl/api/util"
"github.com/tal-tech/go-zero/tools/goctl/config"
"github.com/tal-tech/go-zero/tools/goctl/util"
"github.com/tal-tech/go-zero/tools/goctl/util/format"
@@ -151,10 +150,10 @@ func genRouteImports(parentPkg string, api *spec.ApiSpec) string {
importSet.AddStr(fmt.Sprintf("\"%s\"", util.JoinPackages(parentPkg, contextDir)))
for _, group := range api.Service.Groups {
for _, route := range group.Routes {
folder, ok := apiutil.GetAnnotationValue(route.Annotations, "server", groupProperty)
if !ok {
folder, ok = apiutil.GetAnnotationValue(group.Annotations, "server", groupProperty)
if !ok {
folder := route.GetAnnotation(groupProperty)
if len(folder) == 0 {
folder = group.GetAnnotation(groupProperty)
if len(folder) == 0 {
continue
}
}
@@ -176,12 +175,12 @@ func getRoutes(api *spec.ApiSpec) ([]group, error) {
for _, r := range g.Routes {
handler := getHandlerName(r)
handler = handler + "(serverCtx)"
folder, ok := apiutil.GetAnnotationValue(r.Annotations, "server", groupProperty)
if ok {
folder := r.GetAnnotation(groupProperty)
if len(folder) > 0 {
handler = toPrefix(folder) + "." + strings.ToUpper(handler[:1]) + handler[1:]
} else {
folder, ok = apiutil.GetAnnotationValue(g.Annotations, "server", groupProperty)
if ok {
folder = g.GetAnnotation(groupProperty)
if len(folder) > 0 {
handler = toPrefix(folder) + "." + strings.ToUpper(handler[:1]) + handler[1:]
}
}
@@ -192,12 +191,14 @@ func getRoutes(api *spec.ApiSpec) ([]group, error) {
})
}
if value, ok := apiutil.GetAnnotationValue(g.Annotations, "server", "jwt"); ok {
groupedRoutes.authName = value
jwt := g.GetAnnotation("jwt")
if len(jwt) > 0 {
groupedRoutes.authName = jwt
groupedRoutes.jwtEnabled = true
}
if value, ok := apiutil.GetAnnotationValue(g.Annotations, "server", "middleware"); ok {
for _, item := range strings.Split(value, ",") {
middleware := g.GetAnnotation("middleware")
if len(middleware) > 0 {
for _, item := range strings.Split(middleware, ",") {
groupedRoutes.middlewares = append(groupedRoutes.middlewares, item)
}
}

View File

@@ -35,8 +35,8 @@ func BuildTypes(types []spec.Type) (string, error) {
} else {
builder.WriteString("\n\n")
}
if err := writeType(&builder, tp, types); err != nil {
return "", apiutil.WrapErr(err, "Type "+tp.Name+" generate error")
if err := writeType(&builder, tp); err != nil {
return "", apiutil.WrapErr(err, "Type "+tp.Name()+" generate error")
}
}
@@ -53,6 +53,7 @@ func genTypes(dir string, cfg *config.Config, api *spec.ApiSpec) error {
if err != nil {
return err
}
typeFilename = typeFilename + ".go"
filename := path.Join(dir, typesDir, typeFilename)
os.Remove(filename)
@@ -67,67 +68,28 @@ func genTypes(dir string, cfg *config.Config, api *spec.ApiSpec) error {
builtinTemplate: typesTemplate,
data: map[string]interface{}{
"types": val,
"containsTime": api.ContainsTime(),
"containsTime": false,
},
})
}
func convertTypeCase(types []spec.Type, t string) (string, error) {
ts, err := apiutil.DecomposeType(t)
if err != nil {
return "", err
func writeType(writer io.Writer, tp spec.Type) error {
structType, ok := tp.(spec.DefineStruct)
if !ok {
return errors.New(fmt.Sprintf("unspport struct type: %s", tp.Name()))
}
var defTypes []string
for _, tp := range ts {
for _, typ := range types {
if typ.Name == tp {
defTypes = append(defTypes, tp)
}
}
}
for _, tp := range defTypes {
t = strings.ReplaceAll(t, tp, util.Title(tp))
}
return t, nil
}
func writeType(writer io.Writer, tp spec.Type, types []spec.Type) error {
fmt.Fprintf(writer, "type %s struct {\n", util.Title(tp.Name))
for _, member := range tp.Members {
fmt.Fprintf(writer, "type %s struct {\n", util.Title(tp.Name()))
for _, member := range structType.Members {
if member.IsInline {
var found = false
for _, ty := range types {
if strings.ToLower(ty.Name) == strings.ToLower(member.Name) {
found = true
}
}
if !found {
return errors.New("inline type " + member.Name + " not exist, please correct api file")
}
if _, err := fmt.Fprintf(writer, "%s\n", strings.Title(member.Type)); err != nil {
if _, err := fmt.Fprintf(writer, "%s\n", strings.Title(member.Type.Name())); err != nil {
return err
} else {
continue
}
}
tpString, err := convertTypeCase(types, member.Type)
if err != nil {
return err
}
pm, err := member.GetPropertyName()
if err != nil {
return err
}
if !strings.Contains(pm, "_") {
if strings.Title(member.Name) != strings.Title(pm) {
fmt.Printf("type: %s, property name %s json tag illegal, "+
"should set json tag as `json:\"%s\"` \n", tp.Name, member.Name, util.Untitle(member.Name))
}
}
if err := writeProperty(writer, member.Name, tpString, member.Tag, member.GetComment(), 1); err != nil {
if err := writeProperty(writer, member.Name, member.Tag, member.GetComment(), member.Type, 1); err != nil {
return err
}
}

View File

@@ -72,15 +72,15 @@ func getParentPackage(dir string) (string, error) {
return filepath.ToSlash(filepath.Join(projectCtx.Path, strings.TrimPrefix(projectCtx.WorkDir, projectCtx.Dir))), nil
}
func writeProperty(writer io.Writer, name, tp, tag, comment string, indent int) error {
func writeProperty(writer io.Writer, name, tag, comment string, tp spec.Type, indent int) error {
util.WriteIndent(writer, indent)
var err error
if len(comment) > 0 {
comment = strings.TrimPrefix(comment, "//")
comment = "//" + comment
_, err = fmt.Fprintf(writer, "%s %s %s %s\n", strings.Title(name), tp, tag, comment)
_, err = fmt.Fprintf(writer, "%s %s %s %s\n", strings.Title(name), tp.Name(), tag, comment)
} else {
_, err = fmt.Fprintf(writer, "%s %s %s\n", strings.Title(name), tp, tag)
_, err = fmt.Fprintf(writer, "%s %s %s\n", strings.Title(name), tp.Name(), tag)
}
return err
}
@@ -88,11 +88,13 @@ func writeProperty(writer io.Writer, name, tp, tag, comment string, indent int)
func getAuths(api *spec.ApiSpec) []string {
authNames := collection.NewSet()
for _, g := range api.Service.Groups {
if value, ok := util.GetAnnotationValue(g.Annotations, "server", "jwt"); ok {
authNames.Add(value)
jwt := g.GetAnnotation("jwt")
if len(jwt) > 0 {
authNames.Add(jwt)
}
if value, ok := util.GetAnnotationValue(g.Annotations, "server", "signature"); ok {
authNames.Add(value)
signature := g.GetAnnotation("signature")
if len(signature) > 0 {
authNames.Add(signature)
}
}
return authNames.KeysStr()
@@ -101,8 +103,9 @@ func getAuths(api *spec.ApiSpec) []string {
func getMiddleware(api *spec.ApiSpec) []string {
result := collection.NewSet()
for _, g := range api.Service.Groups {
if value, ok := util.GetAnnotationValue(g.Annotations, "server", "middleware"); ok {
for _, item := range strings.Split(value, ",") {
middleware := g.GetAnnotation("middleware")
if len(middleware) > 0 {
for _, item := range strings.Split(middleware, ",") {
result.Add(strings.TrimSpace(item))
}
}
@@ -118,3 +121,70 @@ func formatCode(code string) string {
return string(ret)
}
func responseGoTypeName(r spec.Route, pkg ...string) string {
if r.ResponseType == nil {
return ""
}
return golangExpr(r.ResponseType, pkg...)
}
func requestGoTypeName(r spec.Route, pkg ...string) string {
if r.RequestType == nil {
return ""
}
return golangExpr(r.RequestType, pkg...)
}
func golangExpr(ty spec.Type, pkg ...string) string {
switch v := ty.(type) {
case spec.PrimitiveType:
return v.RawName
case spec.DefineStruct:
if len(pkg) > 1 {
panic("package cannot be more than 1")
}
if len(pkg) == 0 {
return v.RawName
}
return fmt.Sprintf("%s.%s", pkg[0], strings.Title(v.RawName))
case spec.ArrayType:
if len(pkg) > 1 {
panic("package cannot be more than 1")
}
if len(pkg) == 0 {
return v.RawName
}
return fmt.Sprintf("[]%s", golangExpr(v.Value, pkg...))
case spec.MapType:
if len(pkg) > 1 {
panic("package cannot be more than 1")
}
if len(pkg) == 0 {
return v.RawName
}
return fmt.Sprintf("map[%s]%s", v.Key, golangExpr(v.Value, pkg...))
case spec.PointerType:
if len(pkg) > 1 {
panic("package cannot be more than 1")
}
if len(pkg) == 0 {
return v.RawName
}
return fmt.Sprintf("*%s", golangExpr(v.Type, pkg...))
case spec.InterfaceType:
return v.RawName
}
return ""
}