initial import

This commit is contained in:
kevin
2020-07-26 17:09:05 +08:00
commit 7e3a369a8f
647 changed files with 54754 additions and 0 deletions

View File

@@ -0,0 +1,171 @@
package mongoc
import (
"zero/core/stores/internal"
"zero/core/stores/mongo"
"zero/core/syncx"
"github.com/globalsign/mgo"
)
var (
ErrNotFound = mgo.ErrNotFound
// can't use one SharedCalls per conn, because multiple conns may share the same cache key.
sharedCalls = syncx.NewSharedCalls()
stats = internal.NewCacheStat("mongoc")
)
type (
QueryOption func(query mongo.Query) mongo.Query
cachedCollection struct {
collection mongo.Collection
cache internal.Cache
}
)
func newCollection(collection mongo.Collection, c internal.Cache) *cachedCollection {
return &cachedCollection{
collection: collection,
cache: c,
}
}
func (c *cachedCollection) Count(query interface{}) (int, error) {
return c.collection.Find(query).Count()
}
func (c *cachedCollection) DelCache(keys ...string) error {
return c.cache.DelCache(keys...)
}
func (c *cachedCollection) GetCache(key string, v interface{}) error {
return c.cache.GetCache(key, v)
}
func (c *cachedCollection) FindAllNoCache(v interface{}, query interface{}, opts ...QueryOption) error {
q := c.collection.Find(query)
for _, opt := range opts {
q = opt(q)
}
return q.All(v)
}
func (c *cachedCollection) FindOne(v interface{}, key string, query interface{}) error {
return c.cache.Take(v, key, func(v interface{}) error {
q := c.collection.Find(query)
return q.One(v)
})
}
func (c *cachedCollection) FindOneNoCache(v interface{}, query interface{}) error {
q := c.collection.Find(query)
return q.One(v)
}
func (c *cachedCollection) FindOneId(v interface{}, key string, id interface{}) error {
return c.cache.Take(v, key, func(v interface{}) error {
q := c.collection.FindId(id)
return q.One(v)
})
}
func (c *cachedCollection) FindOneIdNoCache(v interface{}, id interface{}) error {
q := c.collection.FindId(id)
return q.One(v)
}
func (c *cachedCollection) Insert(docs ...interface{}) error {
return c.collection.Insert(docs...)
}
func (c *cachedCollection) Pipe(pipeline interface{}) mongo.Pipe {
return c.collection.Pipe(pipeline)
}
func (c *cachedCollection) Remove(selector interface{}, keys ...string) error {
if err := c.RemoveNoCache(selector); err != nil {
return err
}
return c.DelCache(keys...)
}
func (c *cachedCollection) RemoveNoCache(selector interface{}) error {
return c.collection.Remove(selector)
}
func (c *cachedCollection) RemoveAll(selector interface{}, keys ...string) (*mgo.ChangeInfo, error) {
info, err := c.RemoveAllNoCache(selector)
if err != nil {
return nil, err
}
if err := c.DelCache(keys...); err != nil {
return nil, err
}
return info, nil
}
func (c *cachedCollection) RemoveAllNoCache(selector interface{}) (*mgo.ChangeInfo, error) {
return c.collection.RemoveAll(selector)
}
func (c *cachedCollection) RemoveId(id interface{}, keys ...string) error {
if err := c.RemoveIdNoCache(id); err != nil {
return err
}
return c.DelCache(keys...)
}
func (c *cachedCollection) RemoveIdNoCache(id interface{}) error {
return c.collection.RemoveId(id)
}
func (c *cachedCollection) SetCache(key string, v interface{}) error {
return c.cache.SetCache(key, v)
}
func (c *cachedCollection) Update(selector, update interface{}, keys ...string) error {
if err := c.UpdateNoCache(selector, update); err != nil {
return err
}
return c.DelCache(keys...)
}
func (c *cachedCollection) UpdateNoCache(selector, update interface{}) error {
return c.collection.Update(selector, update)
}
func (c *cachedCollection) UpdateId(id, update interface{}, keys ...string) error {
if err := c.UpdateIdNoCache(id, update); err != nil {
return err
}
return c.DelCache(keys...)
}
func (c *cachedCollection) UpdateIdNoCache(id, update interface{}) error {
return c.collection.UpdateId(id, update)
}
func (c *cachedCollection) Upsert(selector, update interface{}, keys ...string) (*mgo.ChangeInfo, error) {
info, err := c.UpsertNoCache(selector, update)
if err != nil {
return nil, err
}
if err := c.DelCache(keys...); err != nil {
return nil, err
}
return info, nil
}
func (c *cachedCollection) UpsertNoCache(selector, update interface{}) (*mgo.ChangeInfo, error) {
return c.collection.Upsert(selector, update)
}

View File

@@ -0,0 +1,300 @@
package mongoc
import (
"errors"
"io/ioutil"
"log"
"os"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"zero/core/stat"
"zero/core/stores/internal"
"zero/core/stores/mongo"
"zero/core/stores/redis"
"github.com/alicebob/miniredis"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/stretchr/testify/assert"
)
func init() {
stat.SetReporter(nil)
}
func TestStat(t *testing.T) {
resetStats()
s, err := miniredis.Run()
if err != nil {
t.Error(err)
}
r := redis.NewRedis(s.Addr(), redis.NodeType)
cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
c := newCollection(dummyConn{}, cach)
for i := 0; i < 10; i++ {
var str string
if err = c.cache.Take(&str, "name", func(v interface{}) error {
*v.(*string) = "zero"
return nil
}); err != nil {
t.Error(err)
}
}
assert.Equal(t, uint64(10), atomic.LoadUint64(&stats.Total))
assert.Equal(t, uint64(9), atomic.LoadUint64(&stats.Hit))
}
func TestStatCacheFails(t *testing.T) {
resetStats()
log.SetOutput(ioutil.Discard)
defer log.SetOutput(os.Stdout)
r := redis.NewRedis("localhost:59999", redis.NodeType)
cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
c := newCollection(dummyConn{}, cach)
for i := 0; i < 20; i++ {
var str string
err := c.FindOne(&str, "name", bson.M{})
assert.NotNil(t, err)
}
assert.Equal(t, uint64(20), atomic.LoadUint64(&stats.Total))
assert.Equal(t, uint64(0), atomic.LoadUint64(&stats.Hit))
assert.Equal(t, uint64(20), atomic.LoadUint64(&stats.Miss))
assert.Equal(t, uint64(0), atomic.LoadUint64(&stats.DbFails))
}
func TestStatDbFails(t *testing.T) {
resetStats()
s, err := miniredis.Run()
if err != nil {
t.Error(err)
}
r := redis.NewRedis(s.Addr(), redis.NodeType)
cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
c := newCollection(dummyConn{}, cach)
for i := 0; i < 20; i++ {
var str string
err := c.cache.Take(&str, "name", func(v interface{}) error {
return errors.New("db failed")
})
assert.NotNil(t, err)
}
assert.Equal(t, uint64(20), atomic.LoadUint64(&stats.Total))
assert.Equal(t, uint64(0), atomic.LoadUint64(&stats.Hit))
assert.Equal(t, uint64(20), atomic.LoadUint64(&stats.DbFails))
}
func TestStatFromMemory(t *testing.T) {
resetStats()
s, err := miniredis.Run()
if err != nil {
t.Error(err)
}
r := redis.NewRedis(s.Addr(), redis.NodeType)
cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
c := newCollection(dummyConn{}, cach)
var all sync.WaitGroup
var wait sync.WaitGroup
all.Add(10)
wait.Add(4)
go func() {
var str string
if err := c.cache.Take(&str, "name", func(v interface{}) error {
*v.(*string) = "zero"
return nil
}); err != nil {
t.Error(err)
}
wait.Wait()
runtime.Gosched()
all.Done()
}()
for i := 0; i < 4; i++ {
go func() {
var str string
wait.Done()
if err := c.cache.Take(&str, "name", func(v interface{}) error {
*v.(*string) = "zero"
return nil
}); err != nil {
t.Error(err)
}
all.Done()
}()
}
for i := 0; i < 5; i++ {
go func() {
var str string
if err := c.cache.Take(&str, "name", func(v interface{}) error {
*v.(*string) = "zero"
return nil
}); err != nil {
t.Error(err)
}
all.Done()
}()
}
all.Wait()
assert.Equal(t, uint64(10), atomic.LoadUint64(&stats.Total))
assert.Equal(t, uint64(9), atomic.LoadUint64(&stats.Hit))
}
func resetStats() {
atomic.StoreUint64(&stats.Total, 0)
atomic.StoreUint64(&stats.Hit, 0)
atomic.StoreUint64(&stats.Miss, 0)
atomic.StoreUint64(&stats.DbFails, 0)
}
type dummyConn struct {
}
func (c dummyConn) Find(query interface{}) mongo.Query {
return dummyQuery{}
}
func (c dummyConn) FindId(id interface{}) mongo.Query {
return dummyQuery{}
}
func (c dummyConn) Insert(docs ...interface{}) error {
return nil
}
func (c dummyConn) Remove(selector interface{}) error {
return nil
}
func (dummyConn) Pipe(pipeline interface{}) mongo.Pipe {
return nil
}
func (c dummyConn) RemoveAll(selector interface{}) (*mgo.ChangeInfo, error) {
return nil, nil
}
func (c dummyConn) RemoveId(id interface{}) error {
return nil
}
func (c dummyConn) Update(selector, update interface{}) error {
return nil
}
func (c dummyConn) UpdateId(id, update interface{}) error {
return nil
}
func (c dummyConn) Upsert(selector, update interface{}) (*mgo.ChangeInfo, error) {
return nil, nil
}
type dummyQuery struct {
}
func (d dummyQuery) All(result interface{}) error {
return nil
}
func (d dummyQuery) Apply(change mgo.Change, result interface{}) (*mgo.ChangeInfo, error) {
return nil, nil
}
func (d dummyQuery) Count() (int, error) {
return 0, nil
}
func (d dummyQuery) Distinct(key string, result interface{}) error {
return nil
}
func (d dummyQuery) Explain(result interface{}) error {
return nil
}
func (d dummyQuery) For(result interface{}, f func() error) error {
return nil
}
func (d dummyQuery) MapReduce(job *mgo.MapReduce, result interface{}) (*mgo.MapReduceInfo, error) {
return nil, nil
}
func (d dummyQuery) One(result interface{}) error {
return nil
}
func (d dummyQuery) Batch(n int) mongo.Query {
return d
}
func (d dummyQuery) Collation(collation *mgo.Collation) mongo.Query {
return d
}
func (d dummyQuery) Comment(comment string) mongo.Query {
return d
}
func (d dummyQuery) Hint(indexKey ...string) mongo.Query {
return d
}
func (d dummyQuery) Iter() mongo.Iter {
return &mgo.Iter{}
}
func (d dummyQuery) Limit(n int) mongo.Query {
return d
}
func (d dummyQuery) LogReplay() mongo.Query {
return d
}
func (d dummyQuery) Prefetch(p float64) mongo.Query {
return d
}
func (d dummyQuery) Select(selector interface{}) mongo.Query {
return d
}
func (d dummyQuery) SetMaxScan(n int) mongo.Query {
return d
}
func (d dummyQuery) SetMaxTime(duration time.Duration) mongo.Query {
return d
}
func (d dummyQuery) Skip(n int) mongo.Query {
return d
}
func (d dummyQuery) Snapshot() mongo.Query {
return d
}
func (d dummyQuery) Sort(fields ...string) mongo.Query {
return d
}
func (d dummyQuery) Tail(timeout time.Duration) mongo.Iter {
return &mgo.Iter{}
}

View File

@@ -0,0 +1,243 @@
package mongoc
import (
"log"
"zero/core/stores/cache"
"zero/core/stores/internal"
"zero/core/stores/mongo"
"zero/core/stores/redis"
"github.com/globalsign/mgo"
)
type Model struct {
*mongo.Model
cache internal.Cache
generateCollection func(*mgo.Session) *cachedCollection
}
func MustNewNodeModel(url, database, collection string, rds *redis.Redis, opts ...cache.Option) *Model {
model, err := NewNodeModel(url, database, collection, rds, opts...)
if err != nil {
log.Fatal(err)
}
return model
}
func MustNewModel(url, database, collection string, c cache.CacheConf, opts ...cache.Option) *Model {
model, err := NewModel(url, database, collection, c, opts...)
if err != nil {
log.Fatal(err)
}
return model
}
func NewNodeModel(url, database, collection string, rds *redis.Redis, opts ...cache.Option) (*Model, error) {
c := internal.NewCacheNode(rds, sharedCalls, stats, mgo.ErrNotFound, opts...)
return createModel(url, database, collection, c, func(collection mongo.Collection) *cachedCollection {
return newCollection(collection, c)
})
}
func NewModel(url, database, collection string, conf cache.CacheConf, opts ...cache.Option) (*Model, error) {
c := internal.NewCache(conf, sharedCalls, stats, mgo.ErrNotFound, opts...)
return createModel(url, database, collection, c, func(collection mongo.Collection) *cachedCollection {
return newCollection(collection, c)
})
}
func (mm *Model) Count(query interface{}) (int, error) {
return mm.executeInt(func(c *cachedCollection) (int, error) {
return c.Count(query)
})
}
func (mm *Model) DelCache(keys ...string) error {
return mm.cache.DelCache(keys...)
}
func (mm *Model) GetCache(key string, v interface{}) error {
return mm.cache.GetCache(key, v)
}
func (mm *Model) GetCollection(session *mgo.Session) *cachedCollection {
return mm.generateCollection(session)
}
func (mm *Model) FindAllNoCache(v interface{}, query interface{}, opts ...QueryOption) error {
return mm.execute(func(c *cachedCollection) error {
return c.FindAllNoCache(v, query, opts...)
})
}
func (mm *Model) FindOne(v interface{}, key string, query interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.FindOne(v, key, query)
})
}
func (mm *Model) FindOneNoCache(v interface{}, query interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.FindOneNoCache(v, query)
})
}
func (mm *Model) FindOneId(v interface{}, key string, id interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.FindOneId(v, key, id)
})
}
func (mm *Model) FindOneIdNoCache(v interface{}, id interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.FindOneIdNoCache(v, id)
})
}
func (mm *Model) Insert(docs ...interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.Insert(docs...)
})
}
func (mm *Model) Pipe(pipeline interface{}) (mongo.Pipe, error) {
return mm.pipe(func(c *cachedCollection) mongo.Pipe {
return c.Pipe(pipeline)
})
}
func (mm *Model) Remove(selector interface{}, keys ...string) error {
return mm.execute(func(c *cachedCollection) error {
return c.Remove(selector, keys...)
})
}
func (mm *Model) RemoveNoCache(selector interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.RemoveNoCache(selector)
})
}
func (mm *Model) RemoveAll(selector interface{}, keys ...string) (*mgo.ChangeInfo, error) {
return mm.change(func(c *cachedCollection) (*mgo.ChangeInfo, error) {
return c.RemoveAll(selector, keys...)
})
}
func (mm *Model) RemoveAllNoCache(selector interface{}) (*mgo.ChangeInfo, error) {
return mm.change(func(c *cachedCollection) (*mgo.ChangeInfo, error) {
return c.RemoveAllNoCache(selector)
})
}
func (mm *Model) RemoveId(id interface{}, keys ...string) error {
return mm.execute(func(c *cachedCollection) error {
return c.RemoveId(id, keys...)
})
}
func (mm *Model) RemoveIdNoCache(id interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.RemoveIdNoCache(id)
})
}
func (mm *Model) SetCache(key string, v interface{}) error {
return mm.cache.SetCache(key, v)
}
func (mm *Model) Update(selector, update interface{}, keys ...string) error {
return mm.execute(func(c *cachedCollection) error {
return c.Update(selector, update, keys...)
})
}
func (mm *Model) UpdateNoCache(selector, update interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.UpdateNoCache(selector, update)
})
}
func (mm *Model) UpdateId(id, update interface{}, keys ...string) error {
return mm.execute(func(c *cachedCollection) error {
return c.UpdateId(id, update, keys...)
})
}
func (mm *Model) UpdateIdNoCache(id, update interface{}) error {
return mm.execute(func(c *cachedCollection) error {
return c.UpdateIdNoCache(id, update)
})
}
func (mm *Model) Upsert(selector, update interface{}, keys ...string) (*mgo.ChangeInfo, error) {
return mm.change(func(c *cachedCollection) (*mgo.ChangeInfo, error) {
return c.Upsert(selector, update, keys...)
})
}
func (mm *Model) UpsertNoCache(selector, update interface{}) (*mgo.ChangeInfo, error) {
return mm.change(func(c *cachedCollection) (*mgo.ChangeInfo, error) {
return c.UpsertNoCache(selector, update)
})
}
func (mm *Model) change(fn func(c *cachedCollection) (*mgo.ChangeInfo, error)) (*mgo.ChangeInfo, error) {
session, err := mm.TakeSession()
if err != nil {
return nil, err
}
defer mm.PutSession(session)
return fn(mm.GetCollection(session))
}
func (mm *Model) execute(fn func(c *cachedCollection) error) error {
session, err := mm.TakeSession()
if err != nil {
return err
}
defer mm.PutSession(session)
return fn(mm.GetCollection(session))
}
func (mm *Model) executeInt(fn func(c *cachedCollection) (int, error)) (int, error) {
session, err := mm.TakeSession()
if err != nil {
return 0, err
}
defer mm.PutSession(session)
return fn(mm.GetCollection(session))
}
func (mm *Model) pipe(fn func(c *cachedCollection) mongo.Pipe) (mongo.Pipe, error) {
session, err := mm.TakeSession()
if err != nil {
return nil, err
}
defer mm.PutSession(session)
return fn(mm.GetCollection(session)), nil
}
func createModel(url, database, collection string, c internal.Cache,
create func(mongo.Collection) *cachedCollection) (*Model, error) {
model, err := mongo.NewModel(url, database, collection)
if err != nil {
return nil, err
}
return &Model{
Model: model,
cache: c,
generateCollection: func(session *mgo.Session) *cachedCollection {
collection := model.GetCollection(session)
return create(collection)
},
}, nil
}