feature: refactor api parse to g4 (#365)

* feature: refactor api parse to g4 * new g4 parser * add CHANGE_LOG.MD * refactor * fix byte bug * refactor * optimized * optimized * revert * update readme.md * update readme.md * update readme.md * update readme.md * remove no need * fix java gen * add upgrade * resolve confilits Co-authored-by: anqiansong <anqiansong@xiaoheiban.cn>
2021-01-11 15:10:51 +08:00
parent b0ccfb8eb4
commit ee19fb736b
88 changed files with 13641 additions and 2458 deletions
--- a/tools/goctl/api/javagen/gen.go
+++ b/tools/goctl/api/javagen/gen.go
@@ -22,11 +22,7 @@ func JavaCommand(c *cli.Context) error {
 		return errors.New("missing -dir")
 	}

-	p, err := parser.NewParser(apiFile)
-	if err != nil {
-		return err
-	}
-	api, err := p.Parse()
+	api, err := parser.Parse(apiFile)
 	if err != nil {
 		return err
 	}
--- a/tools/goctl/api/javagen/gencomponents.go
+++ b/tools/goctl/api/javagen/gencomponents.go
@@ -8,6 +8,7 @@ import (
 	"strings"
 	"text/template"

+	"github.com/tal-tech/go-zero/core/stringx"
 	"github.com/tal-tech/go-zero/tools/goctl/api/spec"
 	apiutil "github.com/tal-tech/go-zero/tools/goctl/api/util"
 	"github.com/tal-tech/go-zero/tools/goctl/util"
@@ -17,21 +18,46 @@ const (
 	componentTemplate = `// Code generated by goctl. DO NOT EDIT.
 package com.xhb.logic.http.packet.{{.packet}}.model;

-import com.xhb.logic.http.DeProguardable;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+{{.imports}}

 {{.componentType}}
 `
+
+	httpResponseData = "import com.xhb.core.response.HttpResponseData;"
+	httpData         = "import com.xhb.core.packet.HttpData;"
 )

+type componentsContext struct {
+	api           *spec.ApiSpec
+	requestTypes  []spec.Type
+	responseTypes []spec.Type
+	imports       []string
+}
+
 func genComponents(dir, packetName string, api *spec.ApiSpec) error {
-	types := apiutil.GetSharedTypes(api)
+	types := api.Types
 	if len(types) == 0 {
 		return nil
 	}
+
+	var requestTypes []spec.Type
+	var responseTypes []spec.Type
+	for _, group := range api.Service.Groups {
+		for _, route := range group.Routes {
+			if route.RequestType != nil {
+				requestTypes = append(requestTypes, route.RequestType)
+			}
+			if route.ResponseType != nil {
+				responseTypes = append(responseTypes, route.ResponseType)
+			}
+		}
+	}
+
+	context := componentsContext{api: api, requestTypes: requestTypes, responseTypes: responseTypes}
 	for _, ty := range types {
-		if err := createComponent(dir, packetName, ty, api.Types); err != nil {
+		if err := context.createComponent(dir, packetName, ty); err != nil {
 			return err
 		}
 	}
@@ -39,8 +65,21 @@ func genComponents(dir, packetName string, api *spec.ApiSpec) error {
 	return nil
 }

-func createComponent(dir, packetName string, ty spec.Type, types []spec.Type) error {
-	modelFile := util.Title(ty.Name) + ".java"
+func (c *componentsContext) createComponent(dir, packetName string, ty spec.Type) error {
+	defineStruct, ok := ty.(spec.DefineStruct)
+	if !ok {
+		return errors.New("unsupported type %s" + ty.Name())
+	}
+
+	for _, item := range c.requestTypes {
+		if item.Name() == defineStruct.Name() {
+			if len(defineStruct.GetFormMembers())+len(defineStruct.GetBodyMembers()) == 0 {
+				return nil
+			}
+		}
+	}
+
+	modelFile := util.Title(ty.Name()) + ".java"
 	filename := path.Join(dir, modelDir, modelFile)
 	if err := util.RemoveOrQuit(filename); err != nil {
 		return err
@@ -55,77 +94,72 @@ func createComponent(dir, packetName string, ty spec.Type, types []spec.Type) er
 	}
 	defer fp.Close()

-	tys, err := buildType(ty, types)
+	tyString, err := c.buildType(defineStruct)
 	if err != nil {
 		return err
 	}

 	t := template.Must(template.New("componentType").Parse(componentTemplate))
 	return t.Execute(fp, map[string]string{
-		"componentType": tys,
+		"componentType": tyString,
 		"packet":        packetName,
+		"imports":       strings.Join(c.imports, "\n"),
 	})
 }

-func buildType(ty spec.Type, types []spec.Type) (string, error) {
+func (c *componentsContext) buildType(ty spec.DefineStruct) (string, error) {
 	var builder strings.Builder
-	if err := writeType(&builder, ty, types); err != nil {
-		return "", apiutil.WrapErr(err, "Type "+ty.Name+" generate error")
+	if err := c.writeType(&builder, ty); err != nil {
+		return "", apiutil.WrapErr(err, "Type "+ty.Name()+" generate error")
 	}
+
 	return builder.String(), nil
 }

-func writeType(writer io.Writer, tp spec.Type, types []spec.Type) error {
-	fmt.Fprintf(writer, "public class %s implements DeProguardable {\n", util.Title(tp.Name))
-	var members []spec.Member
-	err := writeMembers(writer, types, tp.Members, &members, 1)
+func (c *componentsContext) writeType(writer io.Writer, defineStruct spec.DefineStruct) error {
+	responseData := "HttpData"
+	for _, item := range c.responseTypes {
+		if item.Name() == defineStruct.Name() {
+			responseData = "HttpResponseData"
+			if !stringx.Contains(c.imports, httpResponseData) {
+				c.imports = append(c.imports, httpResponseData)
+			}
+			break
+		}
+	}
+	if responseData == "HttpData" && !stringx.Contains(c.imports, httpData) {
+		c.imports = append(c.imports, httpData)
+	}
+
+	fmt.Fprintf(writer, "public class %s extends %s {\n", util.Title(defineStruct.Name()), responseData)
+	err := c.writeMembers(writer, defineStruct, 1)
 	if err != nil {
 		return err
 	}

-	genGetSet(writer, members, 1)
+	genGetSet(writer, defineStruct, 1)
 	fmt.Fprintf(writer, "}")
 	return nil
 }

-func writeMembers(writer io.Writer, types []spec.Type, members []spec.Member, allMembers *[]spec.Member, indent int) error {
-	for _, member := range members {
-		if !member.IsInline {
-			_, err := member.GetPropertyName()
-			if err != nil {
-				return err
-			}
-		}
-		if !member.IsBodyMember() {
-			continue
-		}
-
-		for _, item := range *allMembers {
-			if item.Name == member.Name {
+func (c *componentsContext) writeMembers(writer io.Writer, ty spec.DefineStruct, indent int) error {
+	for _, member := range ty.Members {
+		if member.IsInline {
+			defineStruct, ok := member.Type.(spec.DefineStruct)
+			if ok {
+				err := c.writeMembers(writer, defineStruct, indent)
+				if err != nil {
+					return err
+				}
 				continue
 			}
+			return errors.New("unsupported inline type %s" + member.Type.Name())
 		}

-		if member.IsInline {
-			hasInline := false
-			for _, ty := range types {
-				if strings.ToLower(ty.Name) == strings.ToLower(member.Name) {
-					err := writeMembers(writer, types, ty.Members, allMembers, indent)
-					if err != nil {
-						return err
-					}
-					hasInline = true
-					break
-				}
-			}
-			if !hasInline {
-				return errors.New("inline type " + member.Name + " not exist, please correct api file")
-			}
-		} else {
+		if member.IsBodyMember() || member.IsFormMember() {
 			if err := writeProperty(writer, member, indent); err != nil {
 				return err
 			}
-			*allMembers = append(*allMembers, member)
 		}
 	}
 	return nil
--- a/tools/goctl/api/javagen/genpacket.go
+++ b/tools/goctl/api/javagen/genpacket.go
@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"strings"
 	"text/template"
@@ -17,29 +16,16 @@ import (

 const packetTemplate = `package com.xhb.logic.http.packet.{{.packet}};

-import com.google.gson.Gson;
-import com.xhb.commons.JSON;
-import com.xhb.commons.JsonMarshal;
+import com.xhb.core.packet.HttpPacket;
 import com.xhb.core.network.HttpRequestClient;
-import com.xhb.core.packet.HttpRequestPacket;
-import com.xhb.core.response.HttpResponseData;
-import com.xhb.logic.http.DeProguardable;
-{{if not .HasRequestBody}}
-import com.xhb.logic.http.request.EmptyRequest;
-{{end}}
-{{.import}}
-
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.json.JSONObject;
-
-public class {{.packetName}} extends HttpRequestPacket<{{.packetName}}.{{.packetName}}Response> {
+{{.imports}}

+public class {{.packetName}} extends HttpPacket<{{.responseType}}> {
 	{{.paramsDeclaration}}

-	public {{.packetName}}({{.params}}{{if .HasRequestBody}}, {{.requestType}} request{{end}}) {
+	public {{.packetName}}({{.params}}{{if .HasRequestBody}}{{.requestType}} request{{end}}) {
 		{{if .HasRequestBody}}super(request);{{else}}super(EmptyRequest.instance);{{end}}
-		{{if .HasRequestBody}}this.request = request;{{end}}{{.paramsSet}}
+		{{if .HasRequestBody}}this.request = request;{{end}}{{.paramsSetter}}
    }

 	@Override
@@ -51,32 +37,6 @@ public class {{.packetName}} extends HttpRequestPacket<{{.packetName}}.{{.packet
    public String requestUri() {
        return {{.uri}};
    }
-
-	@Override
-    public {{.packetName}}Response newInstanceFrom(JSON json) {
-        return new {{.packetName}}Response(json);
-    }
-
-	public static class {{.packetName}}Response extends HttpResponseData {
-
-		private {{.responseType}} responseData;
-
-        {{.packetName}}Response(@NotNull JSON json) {
-            super(json);
-            JSONObject jsonObject = json.asObject();
-			if (JsonParser.hasKey(jsonObject, "data")) {
-				Gson gson = new Gson();
-				JSONObject dataJson = JsonParser.getJSONObject(jsonObject, "data");
-				responseData = gson.fromJson(dataJson.toString(), {{.responseType}}.class);
-			}
-        }
-
-		public {{.responseType}} get{{.responseType}} () {
-            return responseData;
-        }
-    }
-
-	{{.types}}
 }
 `

@@ -91,10 +51,11 @@ func genPacket(dir, packetName string, api *spec.ApiSpec) error {
 }

 func createWith(dir string, api *spec.ApiSpec, route spec.Route, packetName string) error {
-	packet, ok := apiutil.GetAnnotationValue(route.Annotations, "server", "handler")
+	packet := route.Handler
 	packet = strings.Replace(packet, "Handler", "Packet", 1)
-	if !ok {
-		return fmt.Errorf("missing packet annotation for %q", route.Path)
+	packet = strings.Title(packet)
+	if !strings.HasSuffix(packet, "Packet") {
+		packet += "Packet"
 	}

 	javaFile := packet + ".java"
@@ -107,27 +68,24 @@ func createWith(dir string, api *spec.ApiSpec, route spec.Route, packetName stri
 	}
 	defer fp.Close()

-	var builder strings.Builder
-	var first bool
-	tps := apiutil.GetLocalTypes(api, route)
-
-	for _, tp := range tps {
-		if first {
-			first = false
-		} else {
-			fmt.Fprintln(&builder)
-		}
-
-		if err := genType(&builder, tp, api.Types); err != nil {
-			return err
+	var hasRequestBody = false
+	if route.RequestType != nil {
+		if defineStruct, ok := route.RequestType.(spec.DefineStruct); ok {
+			hasRequestBody = len(defineStruct.GetBodyMembers()) > 0 || len(defineStruct.GetFormMembers()) > 0
 		}
 	}
-	types := builder.String()
-	writeIndent(&builder, 1)

-	params := paramsForRoute(route)
+	params := strings.TrimSpace(paramsForRoute(route))
+	if len(params) > 0 && hasRequestBody {
+		params += ", "
+	}
 	paramsDeclaration := declarationForRoute(route)
-	paramsSet := paramsSet(route)
+	paramsSetter := paramsSet(route)
+	imports := getImports(api, packetName)
+
+	if len(route.ResponseTypeName()) == 0 {
+		imports += fmt.Sprintf("\v%s", "import com.xhb.core.response.EmptyResponse;")
+	}

 	t := template.Must(template.New("packetTemplate").Parse(packetTemplate))
 	var tmplBytes bytes.Buffer
@@ -135,15 +93,14 @@ func createWith(dir string, api *spec.ApiSpec, route spec.Route, packetName stri
 		"packetName":        packet,
 		"method":            strings.ToUpper(route.Method),
 		"uri":               processUri(route),
-		"types":             strings.TrimSpace(types),
-		"responseType":      stringx.TakeOne(util.Title(route.ResponseType.Name), "Object"),
+		"responseType":      stringx.TakeOne(util.Title(route.ResponseTypeName()), "EmptyResponse"),
 		"params":            params,
 		"paramsDeclaration": strings.TrimSpace(paramsDeclaration),
-		"paramsSet":         paramsSet,
+		"paramsSetter":      paramsSetter,
 		"packet":            packetName,
-		"requestType":       util.Title(route.RequestType.Name),
-		"HasRequestBody":    len(route.RequestType.GetBodyMembers()) > 0,
-		"import":            getImports(api, route, packetName),
+		"requestType":       util.Title(route.RequestTypeName()),
+		"HasRequestBody":    hasRequestBody,
+		"imports":           imports,
 	})
 	if err != nil {
 		return err
@@ -152,17 +109,11 @@ func createWith(dir string, api *spec.ApiSpec, route spec.Route, packetName stri
 	return nil
 }

-func getImports(api *spec.ApiSpec, route spec.Route, packetName string) string {
+func getImports(api *spec.ApiSpec, packetName string) string {
 	var builder strings.Builder
-	allTypes := apiutil.GetAllTypes(api, route)
-	sharedTypes := apiutil.GetSharedTypes(api)
-	for _, at := range allTypes {
-		for _, item := range sharedTypes {
-			if item.Name == at.Name {
-				fmt.Fprintf(&builder, "import com.xhb.logic.http.packet.%s.model.%s;\n", packetName, item.Name)
-				break
-			}
-		}
+	allTypes := api.Types
+	if len(allTypes) > 0 {
+		fmt.Fprintf(&builder, "import com.xhb.logic.http.packet.%s.model.*;\n", packetName)
 	}
 	return builder.String()
 }
@@ -241,6 +192,7 @@ func declarationForRoute(route spec.Route) string {

 func processUri(route spec.Route) string {
 	path := route.Path
+
 	var builder strings.Builder
 	cops := strings.Split(path, "/")
 	for index, cop := range cops {
@@ -261,29 +213,37 @@ func processUri(route spec.Route) string {
 		result = result[:len(result)-4]
 	}
 	if strings.HasPrefix(result, "/") {
+		result = strings.TrimPrefix(result, "/")
 		result = "\"" + result
 	}
-	return result
+	return result + formString(route)
 }

-func genType(writer io.Writer, tp spec.Type, types []spec.Type) error {
-	if len(tp.GetBodyMembers()) == 0 {
-		return nil
+func formString(route spec.Route) string {
+	var keyValues []string
+	if defineStruct, ok := route.RequestType.(spec.DefineStruct); ok {
+		forms := defineStruct.GetFormMembers()
+		for _, item := range forms {
+			name, err := item.GetPropertyName()
+			if err != nil {
+				panic(err)
+			}
+
+			strcat := "?"
+			if len(keyValues) > 0 {
+				strcat = "&"
+			}
+			if item.Type.Name() == "bool" {
+				name = strings.TrimPrefix(name, "Is")
+				name = strings.TrimPrefix(name, "is")
+				keyValues = append(keyValues, fmt.Sprintf(`"%s%s=" + request.is%s()`, strcat, name, strings.Title(name)))
+			} else {
+				keyValues = append(keyValues, fmt.Sprintf(`"%s%s=" + request.get%s()`, strcat, name, strings.Title(name)))
+			}
+		}
+		if len(keyValues) > 0 {
+			return " + " + strings.Join(keyValues, " + ")
+		}
 	}
-
-	writeIndent(writer, 1)
-	fmt.Fprintf(writer, "static class %s implements DeProguardable {\n", util.Title(tp.Name))
-	var members []spec.Member
-	err := writeMembers(writer, types, tp.Members, &members, 2)
-	if err != nil {
-		return err
-	}
-
-	writeNewline(writer)
-	writeIndent(writer, 1)
-	genGetSet(writer, members, 2)
-	writeIndent(writer, 1)
-	fmt.Fprintln(writer, "}")
-
-	return nil
+	return ""
 }
--- a/tools/goctl/api/javagen/util.go
+++ b/tools/goctl/api/javagen/util.go
@@ -8,19 +8,30 @@ import (
 	"strings"
 	"text/template"

+	"github.com/tal-tech/go-zero/core/stringx"
 	"github.com/tal-tech/go-zero/tools/goctl/api/spec"
-	apiutil "github.com/tal-tech/go-zero/tools/goctl/api/util"
 	"github.com/tal-tech/go-zero/tools/goctl/util"
 )

 const getSetTemplate = `
 {{.indent}}{{.decorator}}
 {{.indent}}public {{.returnType}} get{{.property}}() {
-{{.indent}}	return this.{{.propertyValue}};
+{{.indent}}	return this.{{.tagValue}};
 {{.indent}}}

 {{.indent}}public void set{{.property}}({{.type}} {{.propertyValue}}) {
-{{.indent}}	this.{{.propertyValue}} = {{.propertyValue}};
+{{.indent}}	this.{{.tagValue}} = {{.propertyValue}};
+{{.indent}}}
+`
+
+const boolTemplate = `
+{{.indent}}{{.decorator}}
+{{.indent}}public {{.returnType}} is{{.property}}() {
+{{.indent}}	return this.{{.tagValue}};
+{{.indent}}}
+
+{{.indent}}public void set{{.property}}({{.type}} {{.propertyValue}}) {
+{{.indent}}	this.{{.tagValue}} = {{.propertyValue}};
 {{.indent}}}
 `

@@ -31,22 +42,29 @@ func writeProperty(writer io.Writer, member spec.Member, indent int) error {
 	if err != nil {
 		return err
 	}
+
 	name, err := member.GetPropertyName()
 	if err != nil {
 		return err
 	}
+
 	_, err = fmt.Fprintf(writer, "private %s %s", ty, name)
 	if err != nil {
 		return err
 	}
+
 	writeDefaultValue(writer, member)
 	fmt.Fprint(writer, ";\n")
 	return err
 }

 func writeDefaultValue(writer io.Writer, member spec.Member) error {
-	switch member.Type {
-	case "string":
+	javaType, err := goTypeToJava(member.Type)
+	if err != nil {
+		return err
+	}
+
+	if javaType == "String" {
 		_, err := fmt.Fprintf(writer, " = \"\"")
 		return err
 	}
@@ -67,82 +85,84 @@ func indentString(indent int) string {
 	return result
 }

-func writeNewline(writer io.Writer) {
-	fmt.Fprint(writer, util.NL)
+func goTypeToJava(tp spec.Type) (string, error) {
+	switch v := tp.(type) {
+	case spec.DefineStruct:
+		return util.Title(tp.Name()), nil
+	case spec.PrimitiveType:
+		r, ok := primitiveType(tp.Name())
+		if !ok {
+			return "", errors.New("unsupported primitive type " + tp.Name())
+		}
+		return r, nil
+	case spec.MapType:
+		valueType, err := goTypeToJava(v.Value)
+		if err != nil {
+			return "", err
+		}
+
+		return fmt.Sprintf("java.util.HashMap<String, %s>", util.Title(valueType)), nil
+	case spec.ArrayType:
+		if tp.Name() == "[]byte" {
+			return "byte[]", nil
+		}
+
+		valueType, err := goTypeToJava(v.Value)
+		if err != nil {
+			return "", err
+		}
+
+		return fmt.Sprintf("java.util.ArrayList<%s>", util.Title(valueType)), nil
+	case spec.InterfaceType:
+		return "Object", nil
+	case spec.PointerType:
+		return goTypeToJava(v.Type)
+	}
+
+	return "", errors.New("unsupported primitive type " + tp.Name())
 }

-func isPrimitiveType(tp string) bool {
-	switch tp {
-	case "int", "int32", "int64":
-		return true
-	case "float", "float32", "float64":
-		return true
-	case "bool":
-		return true
-	}
-	return false
-}
-
-func goTypeToJava(tp string) (string, error) {
-	if len(tp) == 0 {
-		return "", errors.New("property type empty")
-	}
-
-	if strings.HasPrefix(tp, "*") {
-		tp = tp[1:]
-	}
+func primitiveType(tp string) (string, bool) {
 	switch tp {
 	case "string":
-		return "String", nil
-	case "int64":
-		return "long", nil
-	case "int", "int8", "int32":
-		return "int", nil
+		return "String", true
+	case "int64", "uint64":
+		return "long", true
+	case "int", "int8", "int32", "uint", "uint8", "uint16", "uint32":
+		return "int", true
 	case "float", "float32", "float64":
-		return "double", nil
+		return "double", true
 	case "bool":
-		return "boolean", nil
+		return "boolean", true
 	}
-	if strings.HasPrefix(tp, "[]") {
-		tys, err := apiutil.DecomposeType(tp)
-		if err != nil {
-			return "", err
-		}

-		if len(tys) == 0 {
-			return "", fmt.Errorf("%s tp parse error", tp)
-		}
-
-		return fmt.Sprintf("java.util.ArrayList<%s>", util.Title(tys[0])), nil
-	} else if strings.HasPrefix(tp, "map") {
-		tys, err := apiutil.DecomposeType(tp)
-		if err != nil {
-			return "", err
-		}
-
-		if len(tys) == 2 {
-			return "", fmt.Errorf("%s tp parse error", tp)
-		}
-
-		return fmt.Sprintf("java.util.HashMap<String, %s>", util.Title(tys[1])), nil
-	}
-	return util.Title(tp), nil
+	return "", false
 }

-func genGetSet(writer io.Writer, members []spec.Member, indent int) error {
-	t := template.Must(template.New("getSetTemplate").Parse(getSetTemplate))
+func genGetSet(writer io.Writer, defineStruct spec.DefineStruct, indent int) error {
+	var members = defineStruct.GetBodyMembers()
+	members = append(members, defineStruct.GetFormMembers()...)
 	for _, member := range members {
-		var tmplBytes bytes.Buffer
-
-		oty, err := goTypeToJava(member.Type)
+		javaType, err := goTypeToJava(member.Type)
 		if err != nil {
-			return err
+			return nil
 		}

-		tyString := oty
+		var property = util.Title(member.Name)
+		var templateStr = getSetTemplate
+		if javaType == "boolean" {
+			templateStr = boolTemplate
+			property = strings.TrimPrefix(property, "Is")
+			property = strings.TrimPrefix(property, "is")
+		}
+		t := template.Must(template.New(templateStr).Parse(getSetTemplate))
+		var tmplBytes bytes.Buffer
+
+		tyString := javaType
 		decorator := ""
-		if !isPrimitiveType(member.Type) {
-			if member.IsOptional() {
+		javaPrimitiveType := []string{"int", "long", "boolean", "float", "double", "short"}
+		if !stringx.Contains(javaPrimitiveType, javaType) {
+			if member.IsOptional() || member.IsOmitEmpty() {
 				decorator = "@Nullable "
 			} else {
 				decorator = "@NotNull "
@@ -150,12 +170,18 @@ func genGetSet(writer io.Writer, members []spec.Member, indent int) error {
 			tyString = decorator + tyString
 		}

+		tagName, err := member.GetPropertyName()
+		if err != nil {
+			return err
+		}
+
 		err = t.Execute(&tmplBytes, map[string]string{
-			"property":      util.Title(member.Name),
+			"property":      property,
 			"propertyValue": util.Untitle(member.Name),
+			"tagValue":      tagName,
 			"type":          tyString,
 			"decorator":     decorator,
-			"returnType":    oty,
+			"returnType":    javaType,
 			"indent":        indentString(indent),
 		})
 		if err != nil {