refactor parser and remove deprecated code (#204)

* add comment support

* add comment support

* 1. group support multi level folder
2. remove force flag

* bug fix

* refactor parser and remove deprecated code

* refactor parser and remove deprecated code

* refactor parser and remove deprecated code

* refactor parser and remove deprecated code

* refactor parser and remove deprecated code

* refactor parser and remove deprecated code

* refactor parser and remove deprecated code

Co-authored-by: kim <xutao@xiaoheiban.cn>

tools/goctl/api/docgen/doc.go

@@ -40,7 +40,7 @@ func genDoc(api *spec.ApiSpec, dir string, filename string) error {
 	defer fp.Close()
 
 	var builder strings.Builder
-	for index, route := range api.Service.Routes {
+	for index, route := range api.Service.Routes() {
 		routeComment, _ := util.GetAnnotationValue(route.Annotations, "doc", "summary")
 		if len(routeComment) == 0 {
 			routeComment = "N/A"

tools/goctl/api/format/format.go

@@ -115,7 +115,7 @@ func apiFormat(data string) (string, error) {
 		return data, nil
 	}
 
-	fs, err := format.Source([]byte(strings.TrimSpace(apiStruct.StructBody)))
+	fs, err := format.Source([]byte(strings.TrimSpace(apiStruct.Type)))
 	if err != nil {
 		str := err.Error()
 		lineNumber := strings.Index(str, ":")

tools/goctl/api/gogen/gen_test.go

@@ -23,7 +23,7 @@ info(
 )
 
 type Request struct {
-  Name string ` + "`" + `path:"name,options=you|me"` + "`" + `
+  Name string ` + "`" + `path:"name,options=you|me"` + "`" + `   // }
 }
 
 type Response struct {
@@ -292,13 +292,13 @@ func TestParser(t *testing.T) {
 	assert.Nil(t, err)
 
 	assert.Equal(t, len(api.Types), 2)
-	assert.Equal(t, len(api.Service.Routes), 2)
+	assert.Equal(t, len(api.Service.Routes()), 2)
 
-	assert.Equal(t, api.Service.Routes[0].Path, "/greet/from/:name")
-	assert.Equal(t, api.Service.Routes[1].Path, "/greet/get")
+	assert.Equal(t, api.Service.Routes()[0].Path, "/greet/from/:name")
+	assert.Equal(t, api.Service.Routes()[1].Path, "/greet/get")
 
-	assert.Equal(t, api.Service.Routes[1].RequestType.Name, "Request")
-	assert.Equal(t, api.Service.Routes[1].ResponseType.Name, "")
+	assert.Equal(t, api.Service.Routes()[1].RequestType.Name, "Request")
+	assert.Equal(t, api.Service.Routes()[1].ResponseType.Name, "")
 
 	validate(t, filename)
 }
@@ -315,7 +315,7 @@ func TestMultiService(t *testing.T) {
 	api, err := parser.Parse()
 	assert.Nil(t, err)
 
-	assert.Equal(t, len(api.Service.Routes), 2)
+	assert.Equal(t, len(api.Service.Routes()), 2)
 	assert.Equal(t, len(api.Service.Groups), 2)
 
 	validate(t, filename)
@@ -342,10 +342,7 @@ func TestInvalidApiFile(t *testing.T) {
 	assert.Nil(t, err)
 	defer os.Remove(filename)
 
-	parser, err := parser.NewParser(filename)
-	assert.Nil(t, err)
-
-	_, err = parser.Parse()
+	_, err = parser.NewParser(filename)
 	assert.NotNil(t, err)
 }
 
@@ -361,8 +358,8 @@ func TestAnonymousAnnotation(t *testing.T) {
 	api, err := parser.Parse()
 	assert.Nil(t, err)
 
-	assert.Equal(t, len(api.Service.Routes), 1)
-	assert.Equal(t, api.Service.Routes[0].Annotations[0].Value, "GreetHandler")
+	assert.Equal(t, len(api.Service.Routes()), 1)
+	assert.Equal(t, api.Service.Routes()[0].Annotations[0].Value, "GreetHandler")
 
 	validate(t, filename)
 }

tools/goctl/api/gogen/genetc.go

@@ -31,11 +31,11 @@ func genEtc(dir string, api *spec.ApiSpec) error {
 	defer fp.Close()
 
 	service := api.Service
-	host, ok := util.GetAnnotationValue(service.Annotations, "server", "host")
+	host, ok := util.GetAnnotationValue(service.Groups[0].Annotations, "server", "host")
 	if !ok {
 		host = "0.0.0.0"
 	}
-	port, ok := util.GetAnnotationValue(service.Annotations, "server", "port")
+	port, ok := util.GetAnnotationValue(service.Groups[0].Annotations, "server", "port")
 	if !ok {
 		port = strconv.Itoa(defaultPort)
 	}

tools/goctl/api/javagen/genpacket.go

@@ -77,7 +77,7 @@ public class {{.packetName}} extends HttpRequestPacket<{{.packetName}}.{{.packet
 `
 
 func genPacket(dir, packetName string, api *spec.ApiSpec) error {
-	for _, route := range api.Service.Routes {
+	for _, route := range api.Service.Routes() {
 		if err := createWith(dir, api, route, packetName); err != nil {
 			return err
 		}

tools/goctl/api/parser/apifileparser.go

@@ -0,0 +1,219 @@
+package parser
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+const (
+	tokenInfo              = "info"
+	tokenImport            = "import"
+	tokenType              = "type"
+	tokenService           = "service"
+	tokenServiceAnnotation = "@server"
+)
+
+type (
+	ApiStruct struct {
+		Info             string
+		Type             string
+		Service          string
+		Imports          string
+		serviceBeginLine int
+	}
+
+	apiFileState interface {
+		process(api *ApiStruct, token string) (apiFileState, error)
+	}
+
+	apiRootState struct {
+		*baseState
+	}
+
+	apiInfoState struct {
+		*baseState
+	}
+
+	apiImportState struct {
+		*baseState
+	}
+
+	apiTypeState struct {
+		*baseState
+	}
+
+	apiServiceState struct {
+		*baseState
+	}
+)
+
+func ParseApi(src string) (*ApiStruct, error) {
+	var buffer = new(bytes.Buffer)
+	buffer.WriteString(src)
+	api := new(ApiStruct)
+	var lineNumber = api.serviceBeginLine
+	apiFile := baseState{r: bufio.NewReader(buffer), lineNumber: &lineNumber}
+	st := apiRootState{&apiFile}
+	for {
+		st, err := st.process(api, "")
+		if err == io.EOF {
+			return api, nil
+		}
+		if err != nil {
+			return nil, fmt.Errorf("near line: %d, %s", lineNumber, err.Error())
+		}
+		if st == nil {
+			return api, nil
+		}
+	}
+}
+
+func (s *apiRootState) process(api *ApiStruct, token string) (apiFileState, error) {
+	var builder strings.Builder
+	for {
+		ch, err := s.readSkipComment()
+		if err != nil {
+			return nil, err
+		}
+
+		switch {
+		case isSpace(ch) || isNewline(ch) || ch == leftParenthesis:
+			token := builder.String()
+			token = strings.TrimSpace(token)
+			if len(token) == 0 {
+				continue
+			}
+
+			builder.Reset()
+			switch token {
+			case tokenInfo:
+				info := apiInfoState{s.baseState}
+				return info.process(api, token+string(ch))
+			case tokenImport:
+				tp := apiImportState{s.baseState}
+				return tp.process(api, token+string(ch))
+			case tokenType:
+				ty := apiTypeState{s.baseState}
+				return ty.process(api, token+string(ch))
+			case tokenService:
+				server := apiServiceState{s.baseState}
+				return server.process(api, token+string(ch))
+			case tokenServiceAnnotation:
+				server := apiServiceState{s.baseState}
+				return server.process(api, token+string(ch))
+			default:
+				if strings.HasPrefix(token, "//") {
+					continue
+				}
+				return nil, errors.New(fmt.Sprintf("invalid token %s at line %d", token, *s.lineNumber))
+			}
+		default:
+			builder.WriteRune(ch)
+		}
+	}
+}
+
+func (s *apiInfoState) process(api *ApiStruct, token string) (apiFileState, error) {
+	for {
+		line, err := s.readLine()
+		if err != nil {
+			return nil, err
+		}
+
+		api.Info += "\n" + token + line
+		token = ""
+		if strings.TrimSpace(line) == string(rightParenthesis) {
+			return &apiRootState{s.baseState}, nil
+		}
+	}
+}
+
+func (s *apiImportState) process(api *ApiStruct, token string) (apiFileState, error) {
+	line, err := s.readLine()
+	if err != nil {
+		return nil, err
+	}
+
+	line = token + line
+	if len(strings.Fields(line)) != 2 {
+		return nil, errors.New("import syntax error: " + line)
+	}
+
+	api.Imports += "\n" + line
+	return &apiRootState{s.baseState}, nil
+}
+
+func (s *apiTypeState) process(api *ApiStruct, token string) (apiFileState, error) {
+	var blockCount = 0
+	for {
+		line, err := s.readLine()
+		if err != nil {
+			return nil, err
+		}
+
+		api.Type += "\n\n" + token + line
+		token = ""
+		line = strings.TrimSpace(line)
+		line = removeComment(line)
+		if strings.HasSuffix(line, leftBrace) {
+			blockCount++
+		}
+		if strings.HasSuffix(line, string(leftParenthesis)) {
+			blockCount++
+		}
+		if strings.HasSuffix(line, string(rightBrace)) {
+			blockCount--
+		}
+		if strings.HasSuffix(line, string(rightParenthesis)) {
+			blockCount--
+		}
+
+		if blockCount == 0 {
+			return &apiRootState{s.baseState}, nil
+		}
+	}
+}
+
+func (s *apiServiceState) process(api *ApiStruct, token string) (apiFileState, error) {
+	var blockCount = 0
+	for {
+		line, err := s.readLineSkipComment()
+		if err != nil {
+			return nil, err
+		}
+
+		line = token + line
+		token = ""
+		api.Service += "\n" + line
+		line = strings.TrimSpace(line)
+		line = removeComment(line)
+		if strings.HasSuffix(line, leftBrace) {
+			blockCount++
+		}
+		if strings.HasSuffix(line, string(leftParenthesis)) {
+			blockCount++
+		}
+		if line == string(rightBrace) {
+			blockCount--
+		}
+		if line == string(rightParenthesis) {
+			blockCount--
+		}
+
+		if blockCount == 0 {
+			return &apiRootState{s.baseState}, nil
+		}
+	}
+}
+
+func removeComment(line string) string {
+	var commentIdx = strings.Index(line, "//")
+	if commentIdx >= 0 {
+		return line[:commentIdx]
+	}
+	return line
+}

tools/goctl/api/parser/parser.go

@@ -3,6 +3,7 @@ package parser
 import (
 	"bufio"
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -34,10 +35,11 @@ func NewParser(filename string) (*Parser, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	for _, item := range strings.Split(apiStruct.Imports, "\n") {
-		ip := strings.TrimSpace(item)
-		if len(ip) > 0 {
-			item := strings.TrimPrefix(item, "import")
+		importLine := strings.TrimSpace(item)
+		if len(importLine) > 0 {
+			item := strings.TrimPrefix(importLine, "import")
 			item = strings.TrimSpace(item)
 			item = strings.TrimPrefix(item, `"`)
 			item = strings.TrimSuffix(item, `"`)
@@ -46,18 +48,33 @@ func NewParser(filename string) (*Parser, error) {
 				path = filepath.Join(filepath.Dir(apiAbsPath), item)
 			}
 			content, err := ioutil.ReadFile(path)
+			if err != nil {
+				return nil, errors.New("import api file not exist: " + item)
+			}
+
+			importStruct, err := ParseApi(string(content))
 			if err != nil {
 				return nil, err
 			}
-			apiStruct.StructBody += "\n" + string(content)
+
+			if len(importStruct.Imports) > 0 {
+				return nil, errors.New("import api should not import another api file recursive")
+			}
+
+			apiStruct.Type += "\n" + importStruct.Type
+			apiStruct.Service += "\n" + importStruct.Service
 		}
 	}
 
+	if len(strings.TrimSpace(apiStruct.Service)) == 0 {
+		return nil, errors.New("api has no service defined")
+	}
+
 	var buffer = new(bytes.Buffer)
 	buffer.WriteString(apiStruct.Service)
 	return &Parser{
 		r:       bufio.NewReader(buffer),
-		typeDef: apiStruct.StructBody,
+		typeDef: apiStruct.Type,
 		api:     apiStruct,
 	}, nil
 }
@@ -69,6 +86,7 @@ func (p *Parser) Parse() (api *spec.ApiSpec, err error) {
 	if err != nil {
 		return nil, err
 	}
+
 	api.Types = types
 	var lineNumber = p.api.serviceBeginLine
 	st := newRootState(p.r, &lineNumber)

tools/goctl/api/parser/servicestate.go

@@ -40,9 +40,7 @@ func (s *serviceState) process(api *spec.ApiSpec) (state, error) {
 	}
 
 	api.Service = spec.Service{
-		Name:        name,
-		Annotations: append(api.Service.Annotations, s.annos...),
-		Routes:      append(api.Service.Routes, routes...),
+		Name: name,
 		Groups: append(api.Service.Groups, spec.Group{
 			Annotations: s.annos,
 			Routes:      routes,

tools/goctl/api/parser/util.go

@@ -2,22 +2,12 @@ package parser
 
 import (
 	"bufio"
-	"errors"
-	"strings"
 
 	"github.com/tal-tech/go-zero/tools/goctl/api/spec"
 )
 
 var emptyType spec.Type
 
-type ApiStruct struct {
-	Info             string
-	StructBody       string
-	Service          string
-	Imports          string
-	serviceBeginLine int
-}
-
 func GetType(api *spec.ApiSpec, t string) spec.Type {
 	for _, tp := range api.Types {
 		if tp.Name == t {
@@ -73,82 +63,3 @@ func skipSpaces(r *bufio.Reader) error {
 func unread(r *bufio.Reader) error {
 	return r.UnreadRune()
 }
-
-func ParseApi(api string) (*ApiStruct, error) {
-	var result ApiStruct
-	scanner := bufio.NewScanner(strings.NewReader(api))
-	var parseInfo = false
-	var parseImport = false
-	var parseType = false
-	var parseService = false
-	var segment string
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-
-		if line == "info(" {
-			parseInfo = true
-		}
-		if line == ")" && parseInfo {
-			parseInfo = false
-			result.Info = segment + ")"
-			segment = ""
-			continue
-		}
-
-		if isImportBeginLine(line) {
-			parseImport = true
-		}
-		if parseImport && (isTypeBeginLine(line) || isServiceBeginLine(line)) {
-			parseImport = false
-			result.Imports = segment
-			segment = line + "\n"
-			continue
-		}
-
-		if isTypeBeginLine(line) {
-			parseType = true
-		}
-		if isServiceBeginLine(line) {
-			parseService = true
-			if parseType {
-				parseType = false
-				result.StructBody = segment
-				segment = line + "\n"
-				continue
-			}
-		}
-		segment += scanner.Text() + "\n"
-	}
-
-	if !parseService {
-		return nil, errors.New("no service defined")
-	}
-	result.Service = segment
-	result.serviceBeginLine = lineBeginOfService(api)
-	return &result, nil
-}
-
-func isImportBeginLine(line string) bool {
-	return strings.HasPrefix(line, "import") && (strings.HasSuffix(line, ".api") || strings.HasSuffix(line, `.api"`))
-}
-
-func isTypeBeginLine(line string) bool {
-	return strings.HasPrefix(line, "type")
-}
-
-func isServiceBeginLine(line string) bool {
-	return strings.HasPrefix(line, "@server") || (strings.HasPrefix(line, "service") && strings.HasSuffix(line, "{"))
-}
-
-func lineBeginOfService(api string) int {
-	scanner := bufio.NewScanner(strings.NewReader(api))
-	var number = 0
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if isServiceBeginLine(line) {
-			break
-		}
-		number++
-	}
-	return number
-}

tools/goctl/api/parser/validator.go

@@ -40,7 +40,7 @@ func (p *Parser) validateDuplicateProperty(tp spec.Type) (bool, string) {
 
 func (p *Parser) validateDuplicateRouteHandler(api *spec.ApiSpec) (bool, string) {
 	var names []string
-	for _, r := range api.Service.Routes {
+	for _, r := range api.Service.Routes() {
 		handler, ok := util.GetAnnotationValue(r.Annotations, "server", "handler")
 		if !ok {
 			return false, fmt.Sprintf("missing handler annotation for %s", r.Path)

tools/goctl/api/spec/fn.go

@@ -27,6 +27,14 @@ type Attribute struct {
 	value string
 }
 
+func (s Service) Routes() []Route {
+	var result []Route
+	for _, group := range s.Groups {
+		result = append(result, group.Routes...)
+	}
+	return result
+}
+
 func (m Member) IsOptional() bool {
 	var option string
 

tools/goctl/api/spec/spec.go

@@ -57,10 +57,8 @@ type (
 	}
 
 	Service struct {
-		Name        string
-		Annotations []Annotation
-		Routes      []Route
-		Groups      []Group
+		Name   string
+		Groups []Group
 	}
 
 	Type struct {

tools/goctl/api/tsgen/genpacket.go

@@ -36,7 +36,7 @@ func genHandler(dir, webApi, caller string, api *spec.ApiSpec, unwrapApi bool) e
 	defer fp.Close()
 
 	var localTypes []spec.Type
-	for _, route := range api.Service.Routes {
+	for _, route := range api.Service.Routes() {
 		rts := apiutil.GetLocalTypes(api, route)
 		localTypes = append(localTypes, rts...)
 	}
@@ -121,7 +121,7 @@ func genTypes(localTypes []spec.Type, inlineType func(string) (*spec.Type, error
 
 func genApi(api *spec.ApiSpec, localTypes []spec.Type, caller string, prefixForType func(string) string) (string, error) {
 	var builder strings.Builder
-	for _, route := range api.Service.Routes {
+	for _, route := range api.Service.Routes() {
 		handler, ok := apiutil.GetAnnotationValue(route.Annotations, "server", "handler")
 		if !ok {
 			return "", fmt.Errorf("missing handler annotation for route %q", route.Path)

tools/goctl/api/util/types.go

@@ -130,7 +130,7 @@ func GetSharedTypes(api *spec.ApiSpec) []spec.Type {
 		}
 		return false
 	}
-	for _, route := range api.Service.Routes {
+	for _, route := range api.Service.Routes() {
 		var rts []spec.Type
 		getTypeRecursive(route.RequestType, types, &rts)
 		getTypeRecursive(route.ResponseType, types, &rts)