feature: refactor api parse to g4 (#365)

* feature: refactor api parse to g4 * new g4 parser * add CHANGE_LOG.MD * refactor * fix byte bug * refactor * optimized * optimized * revert * update readme.md * update readme.md * update readme.md * update readme.md * remove no need * fix java gen * add upgrade * resolve confilits Co-authored-by: anqiansong <anqiansong@xiaoheiban.cn>
2021-01-11 15:10:51 +08:00
parent b0ccfb8eb4
commit ee19fb736b
88 changed files with 13641 additions and 2458 deletions
--- a/tools/goctl/api/tsgen/gen.go
+++ b/tools/goctl/api/tsgen/gen.go
@@ -20,15 +20,12 @@ func TsCommand(c *cli.Context) error {
 	if len(apiFile) == 0 {
 		return errors.New("missing -api")
 	}
+
 	if len(dir) == 0 {
 		return errors.New("missing -dir")
 	}

-	p, err := parser.NewParser(apiFile)
-	if err != nil {
-		return err
-	}
-	api, err := p.Parse()
+	api, err := parser.Parse(apiFile)
 	if err != nil {
 		fmt.Println(aurora.Red("Failed"))
 		return err
--- a/tools/goctl/api/tsgen/gencomponents.go
+++ b/tools/goctl/api/tsgen/gencomponents.go
@@ -1,7 +1,6 @@
 package tsgen

 import (
-	"errors"
 	"path"
 	"strings"
 	"text/template"
@@ -19,19 +18,12 @@ const (
 )

 func genComponents(dir string, api *spec.ApiSpec) error {
-	types := apiutil.GetSharedTypes(api)
+	types := api.Types
 	if len(types) == 0 {
 		return nil
 	}

-	val, err := buildTypes(types, func(name string) (*spec.Type, error) {
-		for _, ty := range api.Types {
-			if strings.ToLower(ty.Name) == strings.ToLower(name) {
-				return &ty, nil
-			}
-		}
-		return nil, errors.New("inline type " + name + " not exist, please correct api file")
-	})
+	val, err := buildTypes(types)
 	if err != nil {
 		return err
 	}
@@ -57,7 +49,7 @@ func genComponents(dir string, api *spec.ApiSpec) error {
 	})
 }

-func buildTypes(types []spec.Type, inlineType func(string) (*spec.Type, error)) (string, error) {
+func buildTypes(types []spec.Type) (string, error) {
 	var builder strings.Builder
 	first := true
 	for _, tp := range types {
@@ -66,12 +58,8 @@ func buildTypes(types []spec.Type, inlineType func(string) (*spec.Type, error))
 		} else {
 			builder.WriteString("\n")
 		}
-		if err := writeType(&builder, tp, func(name string) (*spec.Type, error) {
-			return inlineType(name)
-		}, func(tp string) string {
-			return ""
-		}); err != nil {
-			return "", apiutil.WrapErr(err, "Type "+tp.Name+" generate error")
+		if err := writeType(&builder, tp); err != nil {
+			return "", apiutil.WrapErr(err, "Type "+tp.Name()+" generate error")
 		}
 	}

--- a/tools/goctl/api/tsgen/genpacket.go
+++ b/tools/goctl/api/tsgen/genpacket.go
@@ -1,7 +1,6 @@
 package tsgen

 import (
-	"errors"
 	"fmt"
 	"path"
 	"strings"
@@ -15,8 +14,6 @@ import (
 const (
 	handlerTemplate = `{{.imports}}

-{{.types}}
-
 {{.apis}}
 `
 )
@@ -35,36 +32,6 @@ func genHandler(dir, webApi, caller string, api *spec.ApiSpec, unwrapApi bool) e
 	}
 	defer fp.Close()

-	var localTypes []spec.Type
-	for _, route := range api.Service.Routes() {
-		rts := apiutil.GetLocalTypes(api, route)
-		localTypes = append(localTypes, rts...)
-	}
-
-	var prefixForType = func(ty string) string {
-		if _, pri := primitiveType(ty); pri {
-			return ""
-		}
-		for _, item := range localTypes {
-			if util.Title(item.Name) == ty {
-				return ""
-			}
-		}
-		return packagePrefix
-	}
-
-	types, err := genTypes(localTypes, func(name string) (*spec.Type, error) {
-		for _, ty := range api.Types {
-			if strings.ToLower(ty.Name) == strings.ToLower(name) {
-				return &ty, nil
-			}
-		}
-		return nil, errors.New("inline type " + name + " not exist, please correct api file")
-	}, prefixForType)
-	if err != nil {
-		return err
-	}
-
 	imports := ""
 	if len(caller) == 0 {
 		caller = "webapi"
@@ -76,8 +43,8 @@ func genHandler(dir, webApi, caller string, api *spec.ApiSpec, unwrapApi bool) e
 	if len(webApi) > 0 {
 		imports += `import ` + importCaller + ` from ` + "\"" + webApi + "\""
 	}
-	shardTypes := apiutil.GetSharedTypes(api)
-	if len(shardTypes) != 0 {
+
+	if len(api.Types) != 0 {
 		if len(imports) > 0 {
 			imports += "\n"
 		}
@@ -85,7 +52,7 @@ func genHandler(dir, webApi, caller string, api *spec.ApiSpec, unwrapApi bool) e
 		imports += fmt.Sprintf(`import * as components from "%s"`, "./"+outputFile)
 	}

-	apis, err := genApi(api, caller, prefixForType)
+	apis, err := genApi(api, caller)
 	if err != nil {
 		return err
 	}
@@ -93,54 +60,35 @@ func genHandler(dir, webApi, caller string, api *spec.ApiSpec, unwrapApi bool) e
 	t := template.Must(template.New("handlerTemplate").Parse(handlerTemplate))
 	return t.Execute(fp, map[string]string{
 		"webApi":  webApi,
-		"types":   strings.TrimSpace(types),
 		"imports": imports,
 		"apis":    strings.TrimSpace(apis),
 	})
 }

-func genTypes(localTypes []spec.Type, inlineType func(string) (*spec.Type, error), prefixForType func(string) string) (string, error) {
-	var builder strings.Builder
-	var first bool
-
-	for _, tp := range localTypes {
-		if first {
-			first = false
-		} else {
-			fmt.Fprintln(&builder)
-		}
-		if err := writeType(&builder, tp, func(name string) (s *spec.Type, err error) {
-			return inlineType(name)
-		}, prefixForType); err != nil {
-			return "", err
-		}
-	}
-	types := builder.String()
-	return types, nil
-}
-
-func genApi(api *spec.ApiSpec, caller string, prefixForType func(string) string) (string, error) {
+func genApi(api *spec.ApiSpec, caller string) (string, error) {
 	var builder strings.Builder
 	for _, group := range api.Service.Groups {
 		for _, route := range group.Routes {
-			handler, ok := apiutil.GetAnnotationValue(route.Annotations, "server", "handler")
-			if !ok {
+			handler := route.Handler
+			if len(handler) == 0 {
 				return "", fmt.Errorf("missing handler annotation for route %q", route.Path)
 			}
+
 			handler = util.Untitle(handler)
 			handler = strings.Replace(handler, "Handler", "", 1)
 			comment := commentForRoute(route)
 			if len(comment) > 0 {
 				fmt.Fprintf(&builder, "%s\n", comment)
 			}
-			fmt.Fprintf(&builder, "export function %s(%s) {\n", handler, paramsForRoute(route, prefixForType))
+			fmt.Fprintf(&builder, "export function %s(%s) {\n", handler, paramsForRoute(route))
 			writeIndent(&builder, 1)
 			responseGeneric := "<null>"
-			if len(route.ResponseType.Name) > 0 {
-				val, err := goTypeToTs(route.ResponseType.Name, prefixForType)
+			if len(route.ResponseTypeName()) > 0 {
+				val, err := goTypeToTs(route.ResponseType, true)
 				if err != nil {
 					return "", err
 				}
+
 				responseGeneric = fmt.Sprintf("<%s>", val)
 			}
 			fmt.Fprintf(&builder, `return %s.%s%s(%s)`, caller, strings.ToLower(route.Method),
@@ -153,14 +101,18 @@ func genApi(api *spec.ApiSpec, caller string, prefixForType func(string) string)
 	return apis, nil
 }

-func paramsForRoute(route spec.Route, prefixForType func(string) string) string {
+func paramsForRoute(route spec.Route) string {
+	if route.RequestType == nil {
+		return ""
+	}
 	hasParams := pathHasParams(route)
 	hasBody := hasRequestBody(route)
-	rt, err := goTypeToTs(route.RequestType.Name, prefixForType)
+	rt, err := goTypeToTs(route.RequestType, true)
 	if err != nil {
 		fmt.Println(err.Error())
 		return ""
 	}
+
 	if hasParams && hasBody {
 		return fmt.Sprintf("params: %s, req: %s", rt+"Params", rt)
 	} else if hasParams {
@@ -173,7 +125,7 @@ func paramsForRoute(route spec.Route, prefixForType func(string) string) string

 func commentForRoute(route spec.Route) string {
 	var builder strings.Builder
-	comment, _ := apiutil.GetAnnotationValue(route.Annotations, "doc", "summary")
+	comment := route.JoinedDoc()
 	builder.WriteString("/**")
 	builder.WriteString("\n * @description " + comment)
 	hasParams := pathHasParams(route)
@@ -200,24 +152,35 @@ func callParamsForRoute(route spec.Route, group spec.Group) string {
 	} else if hasBody {
 		return fmt.Sprintf("%s, %s", pathForRoute(route, group), "req")
 	}
+
 	return pathForRoute(route, group)
 }

 func pathForRoute(route spec.Route, group spec.Group) string {
-	value, ok := apiutil.GetAnnotationValue(group.Annotations, "server", pathPrefix)
-	if !ok {
+	prefix := group.GetAnnotation("pathPrefix")
+	if len(prefix) == 0 {
 		return "\"" + route.Path + "\""
 	} else {
-		value = strings.TrimPrefix(value, `"`)
-		value = strings.TrimSuffix(value, `"`)
-		return fmt.Sprintf(`"%s/%s"`, value, strings.TrimPrefix(route.Path, "/"))
+		prefix = strings.TrimPrefix(prefix, `"`)
+		prefix = strings.TrimSuffix(prefix, `"`)
+		return fmt.Sprintf(`"%s/%s"`, prefix, strings.TrimPrefix(route.Path, "/"))
 	}
 }

 func pathHasParams(route spec.Route) bool {
-	return len(route.RequestType.Members) != len(route.RequestType.GetBodyMembers())
+	ds, ok := route.RequestType.(spec.DefineStruct)
+	if !ok {
+		return false
+	}
+
+	return len(ds.Members) != len(ds.GetBodyMembers())
 }

 func hasRequestBody(route spec.Route) bool {
-	return len(route.RequestType.Name) > 0 && len(route.RequestType.GetBodyMembers()) > 0
+	ds, ok := route.RequestType.(spec.DefineStruct)
+	if !ok {
+		return false
+	}
+
+	return len(route.RequestTypeName()) > 0 && len(ds.GetBodyMembers()) > 0
 }
--- a/tools/goctl/api/tsgen/util.go
+++ b/tools/goctl/api/tsgen/util.go
@@ -1,6 +1,7 @@
 package tsgen

 import (
+	"errors"
 	"fmt"
 	"io"
 	"strings"
@@ -10,9 +11,9 @@ import (
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )

-func writeProperty(writer io.Writer, member spec.Member, indent int, prefixForType func(string) string) error {
+func writeProperty(writer io.Writer, member spec.Member, indent int) error {
 	writeIndent(writer, indent)
-	ty, err := goTypeToTs(member.Type, prefixForType)
+	ty, err := goTypeToTs(member.Type, false)
 	if err != nil {
 		return err
 	}
@@ -45,68 +46,56 @@ func writeIndent(writer io.Writer, indent int) {
 	}
 }

-func goTypeToTs(tp string, prefixForType func(string) string) (string, error) {
-	if val, pri := primitiveType(tp); pri {
-		return val, nil
+func goTypeToTs(tp spec.Type, fromPacket bool) (string, error) {
+	switch v := tp.(type) {
+	case spec.DefineStruct:
+		return addPrefix(tp, fromPacket), nil
+	case spec.PrimitiveType:
+		r, ok := primitiveType(tp.Name())
+		if !ok {
+			return "", errors.New("unsupported primitive type " + tp.Name())
+		}
+
+		return r, nil
+	case spec.MapType:
+		valueType, err := goTypeToTs(v.Value, fromPacket)
+		if err != nil {
+			return "", err
+		}
+
+		return fmt.Sprintf("{ [key: string]: %s }", valueType), nil
+	case spec.ArrayType:
+		if tp.Name() == "[]byte" {
+			return "Blob", nil
+		}
+
+		valueType, err := goTypeToTs(v.Value, fromPacket)
+		if err != nil {
+			return "", err
+		}
+
+		return fmt.Sprintf("Array<%s>", valueType), nil
+	case spec.InterfaceType:
+		return "any", nil
+	case spec.PointerType:
+		return goTypeToTs(v.Type, fromPacket)
 	}
-	if tp == "[]byte" {
-		return "Blob", nil
-	} else if strings.HasPrefix(tp, "[][]") {
-		tys, err := apiutil.DecomposeType(tp)
-		if err != nil {
-			return "", err
-		}
-		if len(tys) == 0 {
-			return "", fmt.Errorf("%s tp parse error", tp)
-		}
-		innerType, err := goTypeToTs(tys[0], prefixForType)
-		if err != nil {
-			return "", err
-		}
-		return fmt.Sprintf("Array<Array<%s>>", innerType), nil
-	} else if strings.HasPrefix(tp, "[]") {
-		tys, err := apiutil.DecomposeType(tp)
-		if err != nil {
-			return "", err
-		}
-		if len(tys) == 0 {
-			return "", fmt.Errorf("%s tp parse error", tp)
-		}
-		innerType, err := goTypeToTs(tys[0], prefixForType)
-		if err != nil {
-			return "", err
-		}
-		return fmt.Sprintf("Array<%s>", innerType), nil
-	} else if strings.HasPrefix(tp, "map") {
-		tys, err := apiutil.DecomposeType(tp)
-		if err != nil {
-			return "", err
-		}
-		if len(tys) != 2 {
-			return "", fmt.Errorf("%s tp parse error", tp)
-		}
-		innerType, err := goTypeToTs(tys[1], prefixForType)
-		if err != nil {
-			return "", err
-		}
-		return fmt.Sprintf("{ [key: string]: %s }", innerType), nil
-	}
-	return addPrefixIfNeed(util.Title(tp), prefixForType), nil
+
+	return "", errors.New("unsupported type " + tp.Name())
 }

-func addPrefixIfNeed(tp string, prefixForType func(string) string) string {
-	if val, pri := primitiveType(tp); pri {
-		return val
+func addPrefix(tp spec.Type, fromPacket bool) string {
+	if fromPacket {
+		return packagePrefix + util.Title(tp.Name())
 	}
-	tp = strings.Replace(tp, "*", "", 1)
-	return prefixForType(tp) + util.Title(tp)
+	return util.Title(tp.Name())
 }

 func primitiveType(tp string) (string, bool) {
 	switch tp {
 	case "string":
 		return "string", true
-	case "int", "int8", "int32", "int64":
+	case "int", "int8", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64":
 		return "number", true
 	case "float", "float32", "float64":
 		return "number", true
@@ -120,52 +109,56 @@ func primitiveType(tp string) (string, bool) {
 	return "", false
 }

-func writeType(writer io.Writer, tp spec.Type, inlineType func(string) (*spec.Type, error), prefixForType func(string) string) error {
-	fmt.Fprintf(writer, "export interface %s {\n", util.Title(tp.Name))
-	if err := genMembers(writer, tp, false, inlineType, prefixForType); err != nil {
+func writeType(writer io.Writer, tp spec.Type) error {
+	fmt.Fprintf(writer, "export interface %s {\n", util.Title(tp.Name()))
+	if err := genMembers(writer, tp, false); err != nil {
 		return err
 	}
+
 	fmt.Fprintf(writer, "}\n")
-	err := genParamsTypesIfNeed(writer, tp, inlineType, prefixForType)
-	if err != nil {
-		return err
-	}
-	return nil
+	return genParamsTypesIfNeed(writer, tp)
 }

-func genParamsTypesIfNeed(writer io.Writer, tp spec.Type, inlineType func(string) (*spec.Type, error), prefixForType func(string) string) error {
-	members := tp.GetNonBodyMembers()
+func genParamsTypesIfNeed(writer io.Writer, tp spec.Type) error {
+	definedType, ok := tp.(spec.DefineStruct)
+	if !ok {
+		return errors.New("no members of type " + tp.Name())
+	}
+
+	members := definedType.GetNonBodyMembers()
 	if len(members) == 0 {
 		return nil
 	}
 	fmt.Fprintf(writer, "\n")
-	fmt.Fprintf(writer, "export interface %sParams {\n", util.Title(tp.Name))
-	if err := genMembers(writer, tp, true, inlineType, prefixForType); err != nil {
+	fmt.Fprintf(writer, "export interface %sParams {\n", util.Title(tp.Name()))
+	if err := genMembers(writer, tp, true); err != nil {
 		return err
 	}
+
 	fmt.Fprintf(writer, "}\n")
 	return nil
 }

-func genMembers(writer io.Writer, tp spec.Type, isParam bool, inlineType func(string) (*spec.Type, error), prefixForType func(string) string) error {
-	members := tp.GetBodyMembers()
+func genMembers(writer io.Writer, tp spec.Type, isParam bool) error {
+	definedType, ok := tp.(spec.DefineStruct)
+	if !ok {
+		return errors.New("no members of type " + tp.Name())
+	}
+
+	members := definedType.GetBodyMembers()
 	if isParam {
-		members = tp.GetNonBodyMembers()
+		members = definedType.GetNonBodyMembers()
 	}
 	for _, member := range members {
 		if member.IsInline {
-			// 获取inline类型的成员然后添加到type中
-			it, err := inlineType(strings.TrimPrefix(member.Type, "*"))
-			if err != nil {
-				return err
-			}
-			if err := genMembers(writer, *it, isParam, inlineType, prefixForType); err != nil {
+			if err := genMembers(writer, member.Type, isParam); err != nil {
 				return err
 			}
 			continue
 		}
-		if err := writeProperty(writer, member, 1, prefixForType); err != nil {
-			return apiutil.WrapErr(err, " type "+tp.Name)
+
+		if err := writeProperty(writer, member, 1); err != nil {
+			return apiutil.WrapErr(err, " type "+tp.Name())
 		}
 	}
 	return nil