patch model&rpc (#207)

* change column to read from information_schema

* reactor generate mode from datasource

* reactor generate mode from datasource

* add primary key check logic

* resolve rebase conflicts

* add naming style

* add filename test case

* resolve rebase conflicts

* reactor test

* add test case

* change shell script to makefile

* update rpc new

* update gen_test.go

* format code

* format code

* update test

* generates alias
This commit is contained in:
Keson
2020-11-18 15:32:53 +08:00
committed by GitHub
parent 71083b5e64
commit 24fb29a356
55 changed files with 674 additions and 1163 deletions

View File

@@ -68,30 +68,3 @@ func FieldNames(in interface{}) []string {
}
return out
}
func FieldNamesAlias(in interface{}, alias string) []string {
out := make([]string, 0)
v := reflect.ValueOf(in)
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
// we only accept structs
if v.Kind() != reflect.Struct {
panic(fmt.Errorf("ToMap only accepts structs; got %T", v))
}
typ := v.Type()
for i := 0; i < v.NumField(); i++ {
// gets us a StructField
fi := typ.Field(i)
tagName := ""
if tagv := fi.Tag.Get(dbTag); tagv != "" {
tagName = tagv
} else {
tagName = fi.Name
}
if len(alias) > 0 {
tagName = alias + "." + tagName
}
out = append(out, tagName)
}
return out
}

View File

@@ -17,6 +17,8 @@ import (
"github.com/urfave/cli"
)
var errNotMatched = errors.New("sql not matched")
const (
flagSrc = "src"
flagDir = "dir"
@@ -33,6 +35,20 @@ func MysqlDDL(ctx *cli.Context) error {
cache := ctx.Bool(flagCache)
idea := ctx.Bool(flagIdea)
namingStyle := strings.TrimSpace(ctx.String(flagStyle))
return fromDDl(src, dir, namingStyle, cache, idea)
}
func MyDataSource(ctx *cli.Context) error {
url := strings.TrimSpace(ctx.String(flagUrl))
dir := strings.TrimSpace(ctx.String(flagDir))
cache := ctx.Bool(flagCache)
idea := ctx.Bool(flagIdea)
namingStyle := strings.TrimSpace(ctx.String(flagStyle))
pattern := strings.TrimSpace(ctx.String(flagTable))
return fromDataSource(url, pattern, dir, namingStyle, cache, idea)
}
func fromDDl(src, dir, namingStyle string, cache, idea bool) error {
log := console.NewConsole(idea)
src = strings.TrimSpace(src)
if len(src) == 0 {
@@ -52,29 +68,29 @@ func MysqlDDL(ctx *cli.Context) error {
return err
}
if len(files) == 0 {
return errNotMatched
}
var source []string
for _, file := range files {
data, err := ioutil.ReadFile(file)
if err != nil {
return err
}
source = append(source, string(data))
}
generator := gen.NewDefaultGenerator(strings.Join(source, "\n"), dir, namingStyle, gen.WithConsoleOption(log))
err = generator.Start(cache)
generator, err := gen.NewDefaultGenerator(dir, namingStyle, gen.WithConsoleOption(log))
if err != nil {
log.Error("%v", err)
return err
}
return nil
err = generator.StartFromDDL(strings.Join(source, "\n"), cache)
return err
}
func MyDataSource(ctx *cli.Context) error {
url := strings.TrimSpace(ctx.String(flagUrl))
dir := strings.TrimSpace(ctx.String(flagDir))
cache := ctx.Bool(flagCache)
idea := ctx.Bool(flagIdea)
namingStyle := strings.TrimSpace(ctx.String(flagStyle))
pattern := strings.TrimSpace(ctx.String(flagTable))
func fromDataSource(url, pattern, dir, namingStyle string, cache, idea bool) error {
log := console.NewConsole(idea)
if len(url) == 0 {
log.Error("%v", "expected data source of mysql, but nothing found")
@@ -100,10 +116,8 @@ func MyDataSource(ctx *cli.Context) error {
}
logx.Disable()
conn := sqlx.NewMysql(url)
databaseSource := strings.TrimSuffix(url, "/"+cfg.DBName) + "/information_schema"
db := sqlx.NewMysql(databaseSource)
m := model.NewDDLModel(conn)
im := model.NewInformationSchemaModel(db)
tables, err := im.GetAllTables(cfg.DBName)
@@ -111,7 +125,7 @@ func MyDataSource(ctx *cli.Context) error {
return err
}
var matchTables []string
matchTables := make(map[string][]*model.Column)
for _, item := range tables {
match, err := filepath.Match(pattern, item)
if err != nil {
@@ -121,24 +135,22 @@ func MyDataSource(ctx *cli.Context) error {
if !match {
continue
}
matchTables = append(matchTables, item)
columns, err := im.FindByTableName(cfg.DBName, item)
if err != nil {
return err
}
matchTables[item] = columns
}
if len(matchTables) == 0 {
return errors.New("no tables matched")
}
ddl, err := m.ShowDDL(matchTables...)
generator, err := gen.NewDefaultGenerator(dir, namingStyle, gen.WithConsoleOption(log))
if err != nil {
log.Error("%v", err)
return nil
return err
}
generator := gen.NewDefaultGenerator(strings.Join(ddl, "\n"), dir, namingStyle, gen.WithConsoleOption(log))
err = generator.Start(cache)
if err != nil {
log.Error("%v", err)
}
return nil
err = generator.StartFromInformationSchema(cfg.DBName, matchTables, cache)
return err
}

View File

@@ -0,0 +1,75 @@
package command
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/gen"
"github.com/tal-tech/go-zero/tools/goctl/util"
)
var sql = "-- 用户表 --\nCREATE TABLE `user` (\n `id` bigint(10) NOT NULL AUTO_INCREMENT,\n `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户名称',\n `password` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户密码',\n `mobile` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '手机号',\n `gender` char(5) COLLATE utf8mb4_general_ci NOT NULL COMMENT '男|女|未公开',\n `nickname` varchar(255) COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '用户昵称',\n `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,\n `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n PRIMARY KEY (`id`),\n UNIQUE KEY `name_index` (`name`),\n UNIQUE KEY `mobile_index` (`mobile`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;\n\n"
func TestFromDDl(t *testing.T) {
err := fromDDl("./user.sql", t.TempDir(), gen.NamingCamel, true, false)
assert.Equal(t, errNotMatched, err)
// case dir is not exists
unknownDir := filepath.Join(t.TempDir(), "test", "user.sql")
err = fromDDl(unknownDir, t.TempDir(), gen.NamingCamel, true, false)
assert.True(t, func() bool {
switch err.(type) {
case *os.PathError:
return true
default:
return false
}
}())
// case empty src
err = fromDDl("", t.TempDir(), gen.NamingCamel, true, false)
if err != nil {
assert.Equal(t, "expected path or path globbing patterns, but nothing found", err.Error())
}
// case unknown naming style
tmp := filepath.Join(t.TempDir(), "user.sql")
err = fromDDl(tmp, t.TempDir(), "lower1", true, false)
if err != nil {
assert.Equal(t, "unexpected naming style: lower1", err.Error())
}
tempDir := filepath.Join(t.TempDir(), "test")
err = util.MkdirIfNotExist(tempDir)
if err != nil {
return
}
user1Sql := filepath.Join(tempDir, "user1.sql")
user2Sql := filepath.Join(tempDir, "user2.sql")
err = ioutil.WriteFile(user1Sql, []byte(sql), os.ModePerm)
if err != nil {
return
}
err = ioutil.WriteFile(user2Sql, []byte(sql), os.ModePerm)
if err != nil {
return
}
_, err = os.Stat(user1Sql)
assert.Nil(t, err)
_, err = os.Stat(user2Sql)
assert.Nil(t, err)
err = fromDDl(filepath.Join(tempDir, "user*.sql"), tempDir, gen.NamingLower, true, false)
assert.Nil(t, err)
_, err = os.Stat(filepath.Join(tempDir, "usermodel.go"))
assert.Nil(t, err)
}

View File

@@ -1,11 +0,0 @@
#!/bin/bash
# generate model with cache from ddl
goctl model mysql ddl -src="./sql/*.sql" -dir="./sql/model/user" -c
# generate model with cache from data source
#user=root
#password=password
#datasource=127.0.0.1:3306
#database=test
#goctl model mysql datasource -url="${user}:${password}@tcp(${datasource})/${database}" -table="*" -dir ./model

View File

@@ -0,0 +1,15 @@
#!/bin/bash
# generate model with cache from ddl
fromDDL:
goctl model mysql ddl -src="./sql/*.sql" -dir="./sql/model/user" -c
# generate model with cache from data source
user=root
password=password
datasource=127.0.0.1:3306
database=gozero
fromDataSource:
goctl model mysql datasource -url="$(user):$(password)@tcp($(datasource))/$(database)" -table="*" -dir ./model/cache -c -style camel

View File

@@ -42,5 +42,6 @@ func genDelete(table Table, withCache bool) (string, error) {
if err != nil {
return "", err
}
return output.String(), nil
}

View File

@@ -15,6 +15,7 @@ func genFields(fields []parser.Field) (string, error) {
if err != nil {
return "", err
}
list = append(list, result)
}
return strings.Join(list, "\n"), nil
@@ -43,5 +44,6 @@ func genField(field parser.Field) (string, error) {
if err != nil {
return "", err
}
return output.String(), nil
}

View File

@@ -28,5 +28,6 @@ func genFindOne(table Table, withCache bool) (string, error) {
if err != nil {
return "", err
}
return output.String(), nil
}

View File

@@ -7,6 +7,7 @@ import (
"path/filepath"
"strings"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/model"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/parser"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/template"
"github.com/tal-tech/go-zero/tools/goctl/util"
@@ -24,8 +25,8 @@ const (
type (
defaultGenerator struct {
source string
dir string
//source string
dir string
console.Console
pkg string
namingStyle string
@@ -33,18 +34,30 @@ type (
Option func(generator *defaultGenerator)
)
func NewDefaultGenerator(source, dir, namingStyle string, opt ...Option) *defaultGenerator {
func NewDefaultGenerator(dir, namingStyle string, opt ...Option) (*defaultGenerator, error) {
if dir == "" {
dir = pwd
}
generator := &defaultGenerator{source: source, dir: dir, namingStyle: namingStyle}
dirAbs, err := filepath.Abs(dir)
if err != nil {
return nil, err
}
dir = dirAbs
pkg := filepath.Base(dirAbs)
err = util.MkdirIfNotExist(dir)
if err != nil {
return nil, err
}
generator := &defaultGenerator{dir: dir, namingStyle: namingStyle, pkg: pkg}
var optionList []Option
optionList = append(optionList, newDefaultOption())
optionList = append(optionList, opt...)
for _, fn := range optionList {
fn(generator)
}
return generator
return generator, nil
}
func WithConsoleOption(c console.Console) Option {
@@ -59,21 +72,45 @@ func newDefaultOption() Option {
}
}
func (g *defaultGenerator) Start(withCache bool) error {
func (g *defaultGenerator) StartFromDDL(source string, withCache bool) error {
modelList, err := g.genFromDDL(source, withCache)
if err != nil {
return err
}
return g.createFile(modelList)
}
func (g *defaultGenerator) StartFromInformationSchema(db string, columns map[string][]*model.Column, withCache bool) error {
m := make(map[string]string)
for tableName, column := range columns {
table, err := parser.ConvertColumn(db, tableName, column)
if err != nil {
return err
}
code, err := g.genModel(*table, withCache)
if err != nil {
return err
}
m[table.Name.Source()] = code
}
return g.createFile(m)
}
func (g *defaultGenerator) createFile(modelList map[string]string) error {
dirAbs, err := filepath.Abs(g.dir)
if err != nil {
return err
}
g.dir = dirAbs
g.pkg = filepath.Base(dirAbs)
err = util.MkdirIfNotExist(dirAbs)
if err != nil {
return err
}
modelList, err := g.genFromDDL(withCache)
if err != nil {
return err
}
for tableName, code := range modelList {
tn := stringx.From(tableName)
@@ -96,6 +133,9 @@ func (g *defaultGenerator) Start(withCache bool) error {
}
// generate error file
filename := filepath.Join(dirAbs, "vars.go")
if g.namingStyle == NamingCamel {
filename = filepath.Join(dirAbs, "Vars.go")
}
text, err := util.LoadTemplate(category, errTemplateFile, template.Error)
if err != nil {
return err
@@ -113,8 +153,8 @@ func (g *defaultGenerator) Start(withCache bool) error {
}
// ret1: key-table name,value-code
func (g *defaultGenerator) genFromDDL(withCache bool) (map[string]string, error) {
ddlList := g.split()
func (g *defaultGenerator) genFromDDL(source string, withCache bool) (map[string]string, error) {
ddlList := g.split(source)
m := make(map[string]string)
for _, ddl := range ddlList {
table, err := parser.Parse(ddl)
@@ -139,10 +179,15 @@ type (
)
func (g *defaultGenerator) genModel(in parser.Table, withCache bool) (string, error) {
if len(in.PrimaryKey.Name.Source()) == 0 {
return "", fmt.Errorf("table %s: missing primary key", in.Name.Source())
}
text, err := util.LoadTemplate(category, modelTemplateFile, template.Model)
if err != nil {
return "", err
}
t := util.With("model").
Parse(text).
GoFmt(true)

View File

@@ -22,15 +22,19 @@ func TestCacheModel(t *testing.T) {
defer func() {
_ = os.RemoveAll(dir)
}()
g := NewDefaultGenerator(source, cacheDir, NamingLower)
err := g.Start(true)
g, err := NewDefaultGenerator(cacheDir, NamingCamel)
assert.Nil(t, err)
err = g.StartFromDDL(source, true)
assert.Nil(t, err)
assert.True(t, func() bool {
_, err := os.Stat(filepath.Join(cacheDir, "testuserinfomodel.go"))
_, err := os.Stat(filepath.Join(cacheDir, "TestUserInfoModel.go"))
return err == nil
}())
g = NewDefaultGenerator(source, noCacheDir, NamingLower)
err = g.Start(false)
g, err = NewDefaultGenerator(noCacheDir, NamingLower)
assert.Nil(t, err)
err = g.StartFromDDL(source, false)
assert.Nil(t, err)
assert.True(t, func() bool {
_, err := os.Stat(filepath.Join(noCacheDir, "testuserinfomodel.go"))
@@ -47,15 +51,19 @@ func TestNamingModel(t *testing.T) {
defer func() {
_ = os.RemoveAll(dir)
}()
g := NewDefaultGenerator(source, camelDir, NamingCamel)
err := g.Start(true)
g, err := NewDefaultGenerator(camelDir, NamingCamel)
assert.Nil(t, err)
err = g.StartFromDDL(source, true)
assert.Nil(t, err)
assert.True(t, func() bool {
_, err := os.Stat(filepath.Join(camelDir, "TestUserInfoModel.go"))
return err == nil
}())
g = NewDefaultGenerator(source, snakeDir, NamingSnake)
err = g.Start(true)
g, err = NewDefaultGenerator(snakeDir, NamingSnake)
assert.Nil(t, err)
err = g.StartFromDDL(source, true)
assert.Nil(t, err)
assert.True(t, func() bool {
_, err := os.Stat(filepath.Join(snakeDir, "test_user_info_model.go"))

View File

@@ -17,7 +17,6 @@ func TestGenCacheKeys(t *testing.T) {
Name: stringx.From("id"),
DataBaseType: "bigint",
DataType: "int64",
IsKey: false,
IsPrimaryKey: true,
IsUniqueKey: false,
Comment: "自增id",
@@ -29,7 +28,6 @@ func TestGenCacheKeys(t *testing.T) {
Name: stringx.From("mobile"),
DataBaseType: "varchar",
DataType: "string",
IsKey: false,
IsPrimaryKey: false,
IsUniqueKey: true,
Comment: "手机号",
@@ -38,7 +36,6 @@ func TestGenCacheKeys(t *testing.T) {
Name: stringx.From("name"),
DataBaseType: "varchar",
DataType: "string",
IsKey: false,
IsPrimaryKey: false,
IsUniqueKey: true,
Comment: "姓名",
@@ -47,7 +44,6 @@ func TestGenCacheKeys(t *testing.T) {
Name: stringx.From("createTime"),
DataBaseType: "timestamp",
DataType: "time.Time",
IsKey: false,
IsPrimaryKey: false,
IsUniqueKey: false,
Comment: "创建时间",
@@ -56,7 +52,6 @@ func TestGenCacheKeys(t *testing.T) {
Name: stringx.From("updateTime"),
DataBaseType: "timestamp",
DataType: "time.Time",
IsKey: false,
IsPrimaryKey: false,
IsUniqueKey: false,
Comment: "更新时间",

View File

@@ -4,11 +4,10 @@ import (
"regexp"
)
func (g *defaultGenerator) split() []string {
func (g *defaultGenerator) split(source string) []string {
reg := regexp.MustCompile(createTableFlag)
index := reg.FindAllStringIndex(g.source, -1)
index := reg.FindAllStringIndex(source, -1)
list := make([]string, 0)
source := g.source
for i := len(index) - 1; i >= 0; i-- {
subIndex := index[i]
if len(subIndex) == 0 {

View File

@@ -22,5 +22,6 @@ func genTag(in string) (string, error) {
if err != nil {
return "", err
}
return output.String(), nil
}

View File

@@ -27,6 +27,7 @@ func (m *DDLModel) ShowDDL(table ...string) ([]string, error) {
if err != nil {
return nil, err
}
ddl = append(ddl, resp.DDL)
}
return ddl, nil

View File

@@ -8,6 +8,13 @@ type (
InformationSchemaModel struct {
conn sqlx.SqlConn
}
Column struct {
Name string `db:"COLUMN_NAME"`
DataType string `db:"DATA_TYPE"`
Key string `db:"COLUMN_KEY"`
Extra string `db:"EXTRA"`
Comment string `db:"COLUMN_COMMENT"`
}
)
func NewInformationSchemaModel(conn sqlx.SqlConn) *InformationSchemaModel {
@@ -21,5 +28,13 @@ func (m *InformationSchemaModel) GetAllTables(database string) ([]string, error)
if err != nil {
return nil, err
}
return tables, nil
}
func (m *InformationSchemaModel) FindByTableName(db, table string) ([]*Column, error) {
querySql := `select COLUMN_NAME,DATA_TYPE,COLUMN_KEY,EXTRA,COLUMN_COMMENT from COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME = ?`
var reply []*Column
err := m.conn.QueryRows(&reply, querySql, db, table)
return reply, err
}

View File

@@ -2,8 +2,10 @@ package parser
import (
"fmt"
"strings"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/converter"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/model"
"github.com/tal-tech/go-zero/tools/goctl/util/stringx"
"github.com/xwb1989/sqlparser"
)
@@ -34,7 +36,6 @@ type (
Name stringx.String
DataBaseType string
DataType string
IsKey bool
IsPrimaryKey bool
IsUniqueKey bool
Comment string
@@ -123,7 +124,6 @@ func Parse(ddl string) (*Table, error) {
field.Comment = comment
key, ok := keyMap[column.Name.String()]
if ok {
field.IsKey = true
field.IsPrimaryKey = key == primary
field.IsUniqueKey = key == unique
if field.IsPrimaryKey {
@@ -151,3 +151,62 @@ func (t *Table) ContainsTime() bool {
}
return false
}
func ConvertColumn(db, table string, in []*model.Column) (*Table, error) {
var reply Table
reply.Name = stringx.From(table)
keyMap := make(map[string][]*model.Column)
for _, column := range in {
keyMap[column.Key] = append(keyMap[column.Key], column)
}
primaryColumns := keyMap["PRI"]
if len(primaryColumns) == 0 {
return nil, fmt.Errorf("database:%s, table %s: missing primary key", db, table)
}
if len(primaryColumns) > 1 {
return nil, fmt.Errorf("database:%s, table %s: only one primary key expected", db, table)
}
primaryColumn := primaryColumns[0]
primaryFt, err := converter.ConvertDataType(primaryColumn.DataType)
if err != nil {
return nil, err
}
primaryField := Field{
Name: stringx.From(primaryColumn.Name),
DataBaseType: primaryColumn.DataType,
DataType: primaryFt,
IsUniqueKey: true,
IsPrimaryKey: true,
Comment: primaryColumn.Comment,
}
reply.PrimaryKey = Primary{
Field: primaryField,
AutoIncrement: strings.Contains(primaryColumn.Extra, "auto_increment"),
}
for key, columns := range keyMap {
for _, item := range columns {
dt, err := converter.ConvertDataType(item.DataType)
if err != nil {
return nil, err
}
f := Field{
Name: stringx.From(item.Name),
DataBaseType: item.DataType,
DataType: dt,
IsPrimaryKey: primaryColumn.Name == item.Name,
Comment: item.Comment,
}
if key == "UNI" {
f.IsUniqueKey = true
}
reply.Fields = append(reply.Fields, f)
}
}
return &reply, nil
}

View File

@@ -4,6 +4,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/tal-tech/go-zero/tools/goctl/model/sql/model"
)
func TestParsePlainText(t *testing.T) {
@@ -23,3 +24,58 @@ func TestParseCreateTable(t *testing.T) {
assert.Equal(t, "id", table.PrimaryKey.Name.Source())
assert.Equal(t, true, table.ContainsTime())
}
func TestConvertColumn(t *testing.T) {
_, err := ConvertColumn("user", "user", []*model.Column{
{
Name: "id",
DataType: "bigint",
Key: "",
Extra: "",
Comment: "",
},
})
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "missing primary key")
_, err = ConvertColumn("user", "user", []*model.Column{
{
Name: "id",
DataType: "bigint",
Key: "PRI",
Extra: "",
Comment: "",
},
{
Name: "mobile",
DataType: "varchar",
Key: "PRI",
Extra: "",
Comment: "手机号",
},
})
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "only one primary key expected")
table, err := ConvertColumn("user", "user", []*model.Column{
{
Name: "id",
DataType: "bigint",
Key: "PRI",
Extra: "auto_increment",
Comment: "",
},
{
Name: "mobile",
DataType: "varchar",
Key: "UNI",
Extra: "",
Comment: "手机号",
},
})
assert.Nil(t, err)
assert.True(t, table.PrimaryKey.AutoIncrement && table.PrimaryKey.IsPrimaryKey)
assert.Equal(t, "id", table.PrimaryKey.Name.Source())
assert.Equal(t, "mobile", table.Fields[1].Name.Source())
assert.True(t, table.Fields[1].IsUniqueKey)
}