initial import

core/stores/mongo/bulkinserter.go

@@ -0,0 +1,87 @@
+package mongo
+
+import (
+	"time"
+
+	"zero/core/executors"
+	"zero/core/logx"
+
+	"github.com/globalsign/mgo"
+)
+
+const (
+	flushInterval = time.Second
+	maxBulkRows   = 1000
+)
+
+type (
+	ResultHandler func(*mgo.BulkResult, error)
+
+	BulkInserter struct {
+		executor *executors.PeriodicalExecutor
+		inserter *dbInserter
+	}
+)
+
+func NewBulkInserter(session *mgo.Session, dbName string, collectionNamer func() string) *BulkInserter {
+	inserter := &dbInserter{
+		session:         session,
+		dbName:          dbName,
+		collectionNamer: collectionNamer,
+	}
+
+	return &BulkInserter{
+		executor: executors.NewPeriodicalExecutor(flushInterval, inserter),
+		inserter: inserter,
+	}
+}
+
+func (bi *BulkInserter) Flush() {
+	bi.executor.Flush()
+}
+
+func (bi *BulkInserter) Insert(doc interface{}) {
+	bi.executor.Add(doc)
+}
+
+func (bi *BulkInserter) SetResultHandler(handler ResultHandler) {
+	bi.executor.Sync(func() {
+		bi.inserter.resultHandler = handler
+	})
+}
+
+type dbInserter struct {
+	session         *mgo.Session
+	dbName          string
+	collectionNamer func() string
+	documents       []interface{}
+	resultHandler   ResultHandler
+}
+
+func (in *dbInserter) AddTask(doc interface{}) bool {
+	in.documents = append(in.documents, doc)
+	return len(in.documents) >= maxBulkRows
+}
+
+func (in *dbInserter) Execute(objs interface{}) {
+	docs := objs.([]interface{})
+	if len(docs) == 0 {
+		return
+	}
+
+	bulk := in.session.DB(in.dbName).C(in.collectionNamer()).Bulk()
+	bulk.Insert(docs...)
+	bulk.Unordered()
+	result, err := bulk.Run()
+	if in.resultHandler != nil {
+		in.resultHandler(result, err)
+	} else if err != nil {
+		logx.Error(err)
+	}
+}
+
+func (in *dbInserter) RemoveAll() interface{} {
+	documents := in.documents
+	in.documents = nil
+	return documents
+}

core/stores/mongo/collection.go

@@ -0,0 +1,238 @@
+package mongo
+
+import (
+	"encoding/json"
+	"time"
+
+	"zero/core/breaker"
+	"zero/core/logx"
+	"zero/core/timex"
+
+	"github.com/globalsign/mgo"
+)
+
+const slowThreshold = time.Millisecond * 500
+
+var ErrNotFound = mgo.ErrNotFound
+
+type (
+	Collection interface {
+		Find(query interface{}) Query
+		FindId(id interface{}) Query
+		Insert(docs ...interface{}) error
+		Pipe(pipeline interface{}) Pipe
+		Remove(selector interface{}) error
+		RemoveAll(selector interface{}) (*mgo.ChangeInfo, error)
+		RemoveId(id interface{}) error
+		Update(selector, update interface{}) error
+		UpdateId(id, update interface{}) error
+		Upsert(selector, update interface{}) (*mgo.ChangeInfo, error)
+	}
+
+	decoratedCollection struct {
+		*mgo.Collection
+		brk breaker.Breaker
+	}
+
+	keepablePromise struct {
+		promise breaker.Promise
+		log     func(error)
+	}
+)
+
+func newCollection(collection *mgo.Collection) Collection {
+	return &decoratedCollection{
+		Collection: collection,
+		brk:        breaker.NewBreaker(),
+	}
+}
+
+func (c *decoratedCollection) Find(query interface{}) Query {
+	promise, err := c.brk.Allow()
+	if err != nil {
+		return rejectedQuery{}
+	}
+
+	startTime := timex.Now()
+	return promisedQuery{
+		Query: c.Collection.Find(query),
+		promise: keepablePromise{
+			promise: promise,
+			log: func(err error) {
+				duration := timex.Since(startTime)
+				c.logDuration("find", duration, err, query)
+			},
+		},
+	}
+}
+
+func (c *decoratedCollection) FindId(id interface{}) Query {
+	promise, err := c.brk.Allow()
+	if err != nil {
+		return rejectedQuery{}
+	}
+
+	startTime := timex.Now()
+	return promisedQuery{
+		Query: c.Collection.FindId(id),
+		promise: keepablePromise{
+			promise: promise,
+			log: func(err error) {
+				duration := timex.Since(startTime)
+				c.logDuration("findId", duration, err, id)
+			},
+		},
+	}
+}
+
+func (c *decoratedCollection) Insert(docs ...interface{}) (err error) {
+	return c.brk.DoWithAcceptable(func() error {
+		startTime := timex.Now()
+		defer func() {
+			duration := timex.Since(startTime)
+			c.logDuration("insert", duration, err, docs...)
+		}()
+
+		return c.Collection.Insert(docs...)
+	}, acceptable)
+}
+
+func (c *decoratedCollection) Pipe(pipeline interface{}) Pipe {
+	promise, err := c.brk.Allow()
+	if err != nil {
+		return rejectedPipe{}
+	}
+
+	startTime := timex.Now()
+	return promisedPipe{
+		Pipe: c.Collection.Pipe(pipeline),
+		promise: keepablePromise{
+			promise: promise,
+			log: func(err error) {
+				duration := timex.Since(startTime)
+				c.logDuration("pipe", duration, err, pipeline)
+			},
+		},
+	}
+}
+
+func (c *decoratedCollection) Remove(selector interface{}) (err error) {
+	return c.brk.DoWithAcceptable(func() error {
+		startTime := timex.Now()
+		defer func() {
+			duration := timex.Since(startTime)
+			c.logDuration("remove", duration, err, selector)
+		}()
+
+		return c.Collection.Remove(selector)
+	}, acceptable)
+}
+
+func (c *decoratedCollection) RemoveAll(selector interface{}) (info *mgo.ChangeInfo, err error) {
+	err = c.brk.DoWithAcceptable(func() error {
+		startTime := timex.Now()
+		defer func() {
+			duration := timex.Since(startTime)
+			c.logDuration("removeAll", duration, err, selector)
+		}()
+
+		info, err = c.Collection.RemoveAll(selector)
+		return err
+	}, acceptable)
+
+	return
+}
+
+func (c *decoratedCollection) RemoveId(id interface{}) (err error) {
+	return c.brk.DoWithAcceptable(func() error {
+		startTime := timex.Now()
+		defer func() {
+			duration := timex.Since(startTime)
+			c.logDuration("removeId", duration, err, id)
+		}()
+
+		return c.Collection.RemoveId(id)
+	}, acceptable)
+}
+
+func (c *decoratedCollection) Update(selector, update interface{}) (err error) {
+	return c.brk.DoWithAcceptable(func() error {
+		startTime := timex.Now()
+		defer func() {
+			duration := timex.Since(startTime)
+			c.logDuration("update", duration, err, selector, update)
+		}()
+
+		return c.Collection.Update(selector, update)
+	}, acceptable)
+}
+
+func (c *decoratedCollection) UpdateId(id, update interface{}) (err error) {
+	return c.brk.DoWithAcceptable(func() error {
+		startTime := timex.Now()
+		defer func() {
+			duration := timex.Since(startTime)
+			c.logDuration("updateId", duration, err, id, update)
+		}()
+
+		return c.Collection.UpdateId(id, update)
+	}, acceptable)
+}
+
+func (c *decoratedCollection) Upsert(selector, update interface{}) (info *mgo.ChangeInfo, err error) {
+	err = c.brk.DoWithAcceptable(func() error {
+		startTime := timex.Now()
+		defer func() {
+			duration := timex.Since(startTime)
+			c.logDuration("upsert", duration, err, selector, update)
+		}()
+
+		info, err = c.Collection.Upsert(selector, update)
+		return err
+	}, acceptable)
+
+	return
+}
+
+func (c *decoratedCollection) logDuration(method string, duration time.Duration, err error, docs ...interface{}) {
+	content, e := json.Marshal(docs)
+	if e != nil {
+		logx.Error(err)
+	} else if err != nil {
+		if duration > slowThreshold {
+			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - fail(%s) - %s",
+				c.FullName, method, err.Error(), string(content))
+		} else {
+			logx.WithDuration(duration).Infof("mongo(%s) - %s - fail(%s) - %s",
+				c.FullName, method, err.Error(), string(content))
+		}
+	} else {
+		if duration > slowThreshold {
+			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - ok - %s",
+				c.FullName, method, string(content))
+		} else {
+			logx.WithDuration(duration).Infof("mongo(%s) - %s - ok - %s", c.FullName, method, string(content))
+		}
+	}
+}
+
+func (p keepablePromise) accept(err error) error {
+	p.promise.Accept()
+	p.log(err)
+	return err
+}
+
+func (p keepablePromise) keep(err error) error {
+	if acceptable(err) {
+		p.promise.Accept()
+	} else {
+		p.promise.Reject(err.Error())
+	}
+
+	p.log(err)
+	return err
+}
+
+func acceptable(err error) bool {
+	return err == nil || err == mgo.ErrNotFound
+}

core/stores/mongo/collection_test.go

@@ -0,0 +1,71 @@
+package mongo
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/globalsign/mgo"
+	"github.com/stretchr/testify/assert"
+
+	"zero/core/stringx"
+)
+
+func TestKeepPromise_accept(t *testing.T) {
+	p := new(mockPromise)
+	kp := keepablePromise{
+		promise: p,
+		log:     func(error) {},
+	}
+	assert.Nil(t, kp.accept(nil))
+	assert.Equal(t, mgo.ErrNotFound, kp.accept(mgo.ErrNotFound))
+}
+
+func TestKeepPromise_keep(t *testing.T) {
+	tests := []struct {
+		err      error
+		accepted bool
+		reason   string
+	}{
+		{
+			err:      nil,
+			accepted: true,
+			reason:   "",
+		},
+		{
+			err:      mgo.ErrNotFound,
+			accepted: true,
+			reason:   "",
+		},
+		{
+			err:      errors.New("any"),
+			accepted: false,
+			reason:   "any",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(stringx.RandId(), func(t *testing.T) {
+			p := new(mockPromise)
+			kp := keepablePromise{
+				promise: p,
+				log:     func(error) {},
+			}
+			assert.Equal(t, test.err, kp.keep(test.err))
+			assert.Equal(t, test.accepted, p.accepted)
+			assert.Equal(t, test.reason, p.reason)
+		})
+	}
+}
+
+type mockPromise struct {
+	accepted bool
+	reason   string
+}
+
+func (p *mockPromise) Accept() {
+	p.accepted = true
+}
+
+func (p *mockPromise) Reject(reason string) {
+	p.reason = reason
+}

core/stores/mongo/iter.go

@@ -0,0 +1,96 @@
+//go:generate mockgen -package mongo -destination iter_mock.go -source iter.go Iter
+package mongo
+
+import (
+	"zero/core/breaker"
+
+	"github.com/globalsign/mgo/bson"
+)
+
+type (
+	Iter interface {
+		All(result interface{}) error
+		Close() error
+		Done() bool
+		Err() error
+		For(result interface{}, f func() error) error
+		Next(result interface{}) bool
+		State() (int64, []bson.Raw)
+		Timeout() bool
+	}
+
+	ClosableIter struct {
+		Iter
+		Cleanup func()
+	}
+
+	promisedIter struct {
+		Iter
+		promise keepablePromise
+	}
+
+	rejectedIter struct{}
+)
+
+func (i promisedIter) All(result interface{}) error {
+	return i.promise.keep(i.Iter.All(result))
+}
+
+func (i promisedIter) Close() error {
+	return i.promise.keep(i.Iter.Close())
+}
+
+func (i promisedIter) Err() error {
+	return i.Iter.Err()
+}
+
+func (i promisedIter) For(result interface{}, f func() error) error {
+	var ferr error
+	err := i.Iter.For(result, func() error {
+		ferr = f()
+		return ferr
+	})
+	if ferr == err {
+		return i.promise.accept(err)
+	}
+
+	return i.promise.keep(err)
+}
+
+func (it *ClosableIter) Close() error {
+	err := it.Iter.Close()
+	it.Cleanup()
+	return err
+}
+
+func (i rejectedIter) All(result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (i rejectedIter) Close() error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (i rejectedIter) Done() bool {
+	return false
+}
+
+func (i rejectedIter) Err() error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (i rejectedIter) For(result interface{}, f func() error) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (i rejectedIter) Next(result interface{}) bool {
+	return false
+}
+
+func (i rejectedIter) State() (int64, []bson.Raw) {
+	return 0, nil
+}
+
+func (i rejectedIter) Timeout() bool {
+	return false
+}

core/stores/mongo/iter_mock.go

@@ -0,0 +1,147 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: iter.go
+
+// Package mongo is a generated GoMock package.
+package mongo
+
+import (
+	bson "github.com/globalsign/mgo/bson"
+	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
+)
+
+// MockIter is a mock of Iter interface
+type MockIter struct {
+	ctrl     *gomock.Controller
+	recorder *MockIterMockRecorder
+}
+
+// MockIterMockRecorder is the mock recorder for MockIter
+type MockIterMockRecorder struct {
+	mock *MockIter
+}
+
+// NewMockIter creates a new mock instance
+func NewMockIter(ctrl *gomock.Controller) *MockIter {
+	mock := &MockIter{ctrl: ctrl}
+	mock.recorder = &MockIterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockIter) EXPECT() *MockIterMockRecorder {
+	return m.recorder
+}
+
+// All mocks base method
+func (m *MockIter) All(result interface{}) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "All", result)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// All indicates an expected call of All
+func (mr *MockIterMockRecorder) All(result interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "All", reflect.TypeOf((*MockIter)(nil).All), result)
+}
+
+// Close mocks base method
+func (m *MockIter) Close() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close
+func (mr *MockIterMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockIter)(nil).Close))
+}
+
+// Done mocks base method
+func (m *MockIter) Done() bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Done")
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// Done indicates an expected call of Done
+func (mr *MockIterMockRecorder) Done() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Done", reflect.TypeOf((*MockIter)(nil).Done))
+}
+
+// Err mocks base method
+func (m *MockIter) Err() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Err")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Err indicates an expected call of Err
+func (mr *MockIterMockRecorder) Err() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Err", reflect.TypeOf((*MockIter)(nil).Err))
+}
+
+// For mocks base method
+func (m *MockIter) For(result interface{}, f func() error) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "For", result, f)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// For indicates an expected call of For
+func (mr *MockIterMockRecorder) For(result, f interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "For", reflect.TypeOf((*MockIter)(nil).For), result, f)
+}
+
+// Next mocks base method
+func (m *MockIter) Next(result interface{}) bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Next", result)
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// Next indicates an expected call of Next
+func (mr *MockIterMockRecorder) Next(result interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Next", reflect.TypeOf((*MockIter)(nil).Next), result)
+}
+
+// State mocks base method
+func (m *MockIter) State() (int64, []bson.Raw) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "State")
+	ret0, _ := ret[0].(int64)
+	ret1, _ := ret[1].([]bson.Raw)
+	return ret0, ret1
+}
+
+// State indicates an expected call of State
+func (mr *MockIterMockRecorder) State() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "State", reflect.TypeOf((*MockIter)(nil).State))
+}
+
+// Timeout mocks base method
+func (m *MockIter) Timeout() bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Timeout")
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// Timeout indicates an expected call of Timeout
+func (mr *MockIterMockRecorder) Timeout() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Timeout", reflect.TypeOf((*MockIter)(nil).Timeout))
+}

core/stores/mongo/iter_test.go

@@ -0,0 +1,265 @@
+package mongo
+
+import (
+	"errors"
+	"testing"
+
+	"zero/core/breaker"
+	"zero/core/stringx"
+	"zero/core/syncx"
+
+	"github.com/globalsign/mgo"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestClosableIter_Close(t *testing.T) {
+	errs := []error{
+		nil,
+		mgo.ErrNotFound,
+	}
+
+	for _, err := range errs {
+		t.Run(stringx.RandId(), func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+
+			cleaned := syncx.NewAtomicBool()
+			iter := NewMockIter(ctrl)
+			iter.EXPECT().Close().Return(err)
+			ci := ClosableIter{
+				Iter: iter,
+				Cleanup: func() {
+					cleaned.Set(true)
+				},
+			}
+			assert.Equal(t, err, ci.Close())
+			assert.True(t, cleaned.True())
+		})
+	}
+}
+
+func TestPromisedIter_AllAndClose(t *testing.T) {
+	tests := []struct {
+		err      error
+		accepted bool
+		reason   string
+	}{
+		{
+			err:      nil,
+			accepted: true,
+			reason:   "",
+		},
+		{
+			err:      mgo.ErrNotFound,
+			accepted: true,
+			reason:   "",
+		},
+		{
+			err:      errors.New("any"),
+			accepted: false,
+			reason:   "any",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(stringx.RandId(), func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+
+			iter := NewMockIter(ctrl)
+			iter.EXPECT().All(gomock.Any()).Return(test.err)
+			promise := new(mockPromise)
+			pi := promisedIter{
+				Iter: iter,
+				promise: keepablePromise{
+					promise: promise,
+					log:     func(error) {},
+				},
+			}
+			assert.Equal(t, test.err, pi.All(nil))
+			assert.Equal(t, test.accepted, promise.accepted)
+			assert.Equal(t, test.reason, promise.reason)
+		})
+	}
+
+	for _, test := range tests {
+		t.Run(stringx.RandId(), func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+
+			iter := NewMockIter(ctrl)
+			iter.EXPECT().Close().Return(test.err)
+			promise := new(mockPromise)
+			pi := promisedIter{
+				Iter: iter,
+				promise: keepablePromise{
+					promise: promise,
+					log:     func(error) {},
+				},
+			}
+			assert.Equal(t, test.err, pi.Close())
+			assert.Equal(t, test.accepted, promise.accepted)
+			assert.Equal(t, test.reason, promise.reason)
+		})
+	}
+}
+
+func TestPromisedIter_Err(t *testing.T) {
+	errs := []error{
+		nil,
+		mgo.ErrNotFound,
+	}
+
+	for _, err := range errs {
+		t.Run(stringx.RandId(), func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+
+			iter := NewMockIter(ctrl)
+			iter.EXPECT().Err().Return(err)
+			promise := new(mockPromise)
+			pi := promisedIter{
+				Iter: iter,
+				promise: keepablePromise{
+					promise: promise,
+					log:     func(error) {},
+				},
+			}
+			assert.Equal(t, err, pi.Err())
+		})
+	}
+}
+
+func TestPromisedIter_For(t *testing.T) {
+	tests := []struct {
+		err      error
+		accepted bool
+		reason   string
+	}{
+		{
+			err:      nil,
+			accepted: true,
+			reason:   "",
+		},
+		{
+			err:      mgo.ErrNotFound,
+			accepted: true,
+			reason:   "",
+		},
+		{
+			err:      errors.New("any"),
+			accepted: false,
+			reason:   "any",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(stringx.RandId(), func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+
+			iter := NewMockIter(ctrl)
+			iter.EXPECT().For(gomock.Any(), gomock.Any()).Return(test.err)
+			promise := new(mockPromise)
+			pi := promisedIter{
+				Iter: iter,
+				promise: keepablePromise{
+					promise: promise,
+					log:     func(error) {},
+				},
+			}
+			assert.Equal(t, test.err, pi.For(nil, nil))
+			assert.Equal(t, test.accepted, promise.accepted)
+			assert.Equal(t, test.reason, promise.reason)
+		})
+	}
+}
+
+func TestRejectedIter_All(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedIter).All(nil))
+}
+
+func TestRejectedIter_Close(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedIter).Close())
+}
+
+func TestRejectedIter_Done(t *testing.T) {
+	assert.False(t, new(rejectedIter).Done())
+}
+
+func TestRejectedIter_Err(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedIter).Err())
+}
+
+func TestRejectedIter_For(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedIter).For(nil, nil))
+}
+
+func TestRejectedIter_Next(t *testing.T) {
+	assert.False(t, new(rejectedIter).Next(nil))
+}
+
+func TestRejectedIter_State(t *testing.T) {
+	n, raw := new(rejectedIter).State()
+	assert.Equal(t, int64(0), n)
+	assert.Nil(t, raw)
+}
+
+func TestRejectedIter_Timeout(t *testing.T) {
+	assert.False(t, new(rejectedIter).Timeout())
+}
+
+func TestIter_Done(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	iter := NewMockIter(ctrl)
+	iter.EXPECT().Done().Return(true)
+	ci := ClosableIter{
+		Iter:    iter,
+		Cleanup: nil,
+	}
+	assert.True(t, ci.Done())
+}
+
+func TestIter_Next(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	iter := NewMockIter(ctrl)
+	iter.EXPECT().Next(gomock.Any()).Return(true)
+	ci := ClosableIter{
+		Iter:    iter,
+		Cleanup: nil,
+	}
+	assert.True(t, ci.Next(nil))
+}
+
+func TestIter_State(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	iter := NewMockIter(ctrl)
+	iter.EXPECT().State().Return(int64(1), nil)
+	ci := ClosableIter{
+		Iter:    iter,
+		Cleanup: nil,
+	}
+	n, raw := ci.State()
+	assert.Equal(t, int64(1), n)
+	assert.Nil(t, raw)
+}
+
+func TestIter_Timeout(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	iter := NewMockIter(ctrl)
+	iter.EXPECT().Timeout().Return(true)
+	ci := ClosableIter{
+		Iter:    iter,
+		Cleanup: nil,
+	}
+	assert.True(t, ci.Timeout())
+}

core/stores/mongo/model.go

@@ -0,0 +1,164 @@
+package mongo
+
+import (
+	"log"
+	"time"
+
+	"github.com/globalsign/mgo"
+)
+
+type (
+	options struct {
+		timeout time.Duration
+	}
+
+	Option func(opts *options)
+
+	Model struct {
+		session    *concurrentSession
+		db         *mgo.Database
+		collection string
+		opts       []Option
+	}
+)
+
+func MustNewModel(url, database, collection string, opts ...Option) *Model {
+	model, err := NewModel(url, database, collection, opts...)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return model
+}
+
+func NewModel(url, database, collection string, opts ...Option) (*Model, error) {
+	session, err := getConcurrentSession(url)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Model{
+		session:    session,
+		db:         session.DB(database),
+		collection: collection,
+		opts:       opts,
+	}, nil
+}
+
+func (mm *Model) Find(query interface{}) (Query, error) {
+	return mm.query(func(c Collection) Query {
+		return c.Find(query)
+	})
+}
+
+func (mm *Model) FindId(id interface{}) (Query, error) {
+	return mm.query(func(c Collection) Query {
+		return c.FindId(id)
+	})
+}
+
+func (mm *Model) GetCollection(session *mgo.Session) Collection {
+	return newCollection(mm.db.C(mm.collection).With(session))
+}
+
+func (mm *Model) Insert(docs ...interface{}) error {
+	return mm.execute(func(c Collection) error {
+		return c.Insert(docs...)
+	})
+}
+
+func (mm *Model) Pipe(pipeline interface{}) (Pipe, error) {
+	return mm.pipe(func(c Collection) Pipe {
+		return c.Pipe(pipeline)
+	})
+}
+
+func (mm *Model) PutSession(session *mgo.Session) {
+	mm.session.putSession(session)
+}
+
+func (mm *Model) Remove(selector interface{}) error {
+	return mm.execute(func(c Collection) error {
+		return c.Remove(selector)
+	})
+}
+
+func (mm *Model) RemoveAll(selector interface{}) (*mgo.ChangeInfo, error) {
+	return mm.change(func(c Collection) (*mgo.ChangeInfo, error) {
+		return c.RemoveAll(selector)
+	})
+}
+
+func (mm *Model) RemoveId(id interface{}) error {
+	return mm.execute(func(c Collection) error {
+		return c.RemoveId(id)
+	})
+}
+
+func (mm *Model) TakeSession() (*mgo.Session, error) {
+	return mm.session.takeSession(mm.opts...)
+}
+
+func (mm *Model) Update(selector, update interface{}) error {
+	return mm.execute(func(c Collection) error {
+		return c.Update(selector, update)
+	})
+}
+
+func (mm *Model) UpdateId(id, update interface{}) error {
+	return mm.execute(func(c Collection) error {
+		return c.UpdateId(id, update)
+	})
+}
+
+func (mm *Model) Upsert(selector, update interface{}) (*mgo.ChangeInfo, error) {
+	return mm.change(func(c Collection) (*mgo.ChangeInfo, error) {
+		return c.Upsert(selector, update)
+	})
+}
+
+func (mm *Model) change(fn func(c Collection) (*mgo.ChangeInfo, error)) (*mgo.ChangeInfo, error) {
+	session, err := mm.TakeSession()
+	if err != nil {
+		return nil, err
+	}
+	defer mm.PutSession(session)
+
+	return fn(mm.GetCollection(session))
+}
+
+func (mm *Model) execute(fn func(c Collection) error) error {
+	session, err := mm.TakeSession()
+	if err != nil {
+		return err
+	}
+	defer mm.PutSession(session)
+
+	return fn(mm.GetCollection(session))
+}
+
+func (mm *Model) pipe(fn func(c Collection) Pipe) (Pipe, error) {
+	session, err := mm.TakeSession()
+	if err != nil {
+		return nil, err
+	}
+	defer mm.PutSession(session)
+
+	return fn(mm.GetCollection(session)), nil
+}
+
+func (mm *Model) query(fn func(c Collection) Query) (Query, error) {
+	session, err := mm.TakeSession()
+	if err != nil {
+		return nil, err
+	}
+	defer mm.PutSession(session)
+
+	return fn(mm.GetCollection(session)), nil
+}
+
+func WithTimeout(timeout time.Duration) Option {
+	return func(opts *options) {
+		opts.timeout = timeout
+	}
+}

core/stores/mongo/pipe.go

@@ -0,0 +1,100 @@
+package mongo
+
+import (
+	"time"
+
+	"zero/core/breaker"
+
+	"github.com/globalsign/mgo"
+)
+
+type (
+	Pipe interface {
+		All(result interface{}) error
+		AllowDiskUse() Pipe
+		Batch(n int) Pipe
+		Collation(collation *mgo.Collation) Pipe
+		Explain(result interface{}) error
+		Iter() Iter
+		One(result interface{}) error
+		SetMaxTime(d time.Duration) Pipe
+	}
+
+	promisedPipe struct {
+		*mgo.Pipe
+		promise keepablePromise
+	}
+
+	rejectedPipe struct{}
+)
+
+func (p promisedPipe) All(result interface{}) error {
+	return p.promise.keep(p.Pipe.All(result))
+}
+
+func (p promisedPipe) AllowDiskUse() Pipe {
+	p.Pipe.AllowDiskUse()
+	return p
+}
+
+func (p promisedPipe) Batch(n int) Pipe {
+	p.Pipe.Batch(n)
+	return p
+}
+
+func (p promisedPipe) Collation(collation *mgo.Collation) Pipe {
+	p.Pipe.Collation(collation)
+	return p
+}
+
+func (p promisedPipe) Explain(result interface{}) error {
+	return p.promise.keep(p.Pipe.Explain(result))
+}
+
+func (p promisedPipe) Iter() Iter {
+	return promisedIter{
+		Iter:    p.Pipe.Iter(),
+		promise: p.promise,
+	}
+}
+
+func (p promisedPipe) One(result interface{}) error {
+	return p.promise.keep(p.Pipe.One(result))
+}
+
+func (p promisedPipe) SetMaxTime(d time.Duration) Pipe {
+	p.Pipe.SetMaxTime(d)
+	return p
+}
+
+func (p rejectedPipe) All(result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (p rejectedPipe) AllowDiskUse() Pipe {
+	return p
+}
+
+func (p rejectedPipe) Batch(n int) Pipe {
+	return p
+}
+
+func (p rejectedPipe) Collation(collation *mgo.Collation) Pipe {
+	return p
+}
+
+func (p rejectedPipe) Explain(result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (p rejectedPipe) Iter() Iter {
+	return rejectedIter{}
+}
+
+func (p rejectedPipe) One(result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (p rejectedPipe) SetMaxTime(d time.Duration) Pipe {
+	return p
+}

core/stores/mongo/pipe_test.go

@@ -0,0 +1,45 @@
+package mongo
+
+import (
+	"testing"
+
+	"zero/core/breaker"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRejectedPipe_All(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedPipe).All(nil))
+}
+
+func TestRejectedPipe_AllowDiskUse(t *testing.T) {
+	var p rejectedPipe
+	assert.Equal(t, p, p.AllowDiskUse())
+}
+
+func TestRejectedPipe_Batch(t *testing.T) {
+	var p rejectedPipe
+	assert.Equal(t, p, p.Batch(1))
+}
+
+func TestRejectedPipe_Collation(t *testing.T) {
+	var p rejectedPipe
+	assert.Equal(t, p, p.Collation(nil))
+}
+
+func TestRejectedPipe_Explain(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedPipe).Explain(nil))
+}
+
+func TestRejectedPipe_Iter(t *testing.T) {
+	assert.EqualValues(t, rejectedIter{}, new(rejectedPipe).Iter())
+}
+
+func TestRejectedPipe_One(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedPipe).One(nil))
+}
+
+func TestRejectedPipe_SetMaxTime(t *testing.T) {
+	var p rejectedPipe
+	assert.Equal(t, p, p.SetMaxTime(0))
+}

core/stores/mongo/query.go

@@ -0,0 +1,285 @@
+package mongo
+
+import (
+	"time"
+
+	"zero/core/breaker"
+
+	"github.com/globalsign/mgo"
+)
+
+type (
+	Query interface {
+		All(result interface{}) error
+		Apply(change mgo.Change, result interface{}) (*mgo.ChangeInfo, error)
+		Batch(n int) Query
+		Collation(collation *mgo.Collation) Query
+		Comment(comment string) Query
+		Count() (int, error)
+		Distinct(key string, result interface{}) error
+		Explain(result interface{}) error
+		For(result interface{}, f func() error) error
+		Hint(indexKey ...string) Query
+		Iter() Iter
+		Limit(n int) Query
+		LogReplay() Query
+		MapReduce(job *mgo.MapReduce, result interface{}) (*mgo.MapReduceInfo, error)
+		One(result interface{}) error
+		Prefetch(p float64) Query
+		Select(selector interface{}) Query
+		SetMaxScan(n int) Query
+		SetMaxTime(d time.Duration) Query
+		Skip(n int) Query
+		Snapshot() Query
+		Sort(fields ...string) Query
+		Tail(timeout time.Duration) Iter
+	}
+
+	promisedQuery struct {
+		*mgo.Query
+		promise keepablePromise
+	}
+
+	rejectedQuery struct{}
+)
+
+func (q promisedQuery) All(result interface{}) error {
+	return q.promise.keep(q.Query.All(result))
+}
+
+func (q promisedQuery) Apply(change mgo.Change, result interface{}) (*mgo.ChangeInfo, error) {
+	info, err := q.Query.Apply(change, result)
+	return info, q.promise.keep(err)
+}
+
+func (q promisedQuery) Batch(n int) Query {
+	return promisedQuery{
+		Query:   q.Query.Batch(n),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Collation(collation *mgo.Collation) Query {
+	return promisedQuery{
+		Query:   q.Query.Collation(collation),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Comment(comment string) Query {
+	return promisedQuery{
+		Query:   q.Query.Comment(comment),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Count() (int, error) {
+	v, err := q.Query.Count()
+	return v, q.promise.keep(err)
+}
+
+func (q promisedQuery) Distinct(key string, result interface{}) error {
+	return q.promise.keep(q.Query.Distinct(key, result))
+}
+
+func (q promisedQuery) Explain(result interface{}) error {
+	return q.promise.keep(q.Query.Explain(result))
+}
+
+func (q promisedQuery) For(result interface{}, f func() error) error {
+	var ferr error
+	err := q.Query.For(result, func() error {
+		ferr = f()
+		return ferr
+	})
+	if ferr == err {
+		return q.promise.accept(err)
+	}
+
+	return q.promise.keep(err)
+}
+
+func (q promisedQuery) Hint(indexKey ...string) Query {
+	return promisedQuery{
+		Query:   q.Query.Hint(indexKey...),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Iter() Iter {
+	return promisedIter{
+		Iter:    q.Query.Iter(),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Limit(n int) Query {
+	return promisedQuery{
+		Query:   q.Query.Limit(n),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) LogReplay() Query {
+	return promisedQuery{
+		Query:   q.Query.LogReplay(),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) MapReduce(job *mgo.MapReduce, result interface{}) (*mgo.MapReduceInfo, error) {
+	info, err := q.Query.MapReduce(job, result)
+	return info, q.promise.keep(err)
+}
+
+func (q promisedQuery) One(result interface{}) error {
+	return q.promise.keep(q.Query.One(result))
+}
+
+func (q promisedQuery) Prefetch(p float64) Query {
+	return promisedQuery{
+		Query:   q.Query.Prefetch(p),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Select(selector interface{}) Query {
+	return promisedQuery{
+		Query:   q.Query.Select(selector),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) SetMaxScan(n int) Query {
+	return promisedQuery{
+		Query:   q.Query.SetMaxScan(n),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) SetMaxTime(d time.Duration) Query {
+	return promisedQuery{
+		Query:   q.Query.SetMaxTime(d),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Skip(n int) Query {
+	return promisedQuery{
+		Query:   q.Query.Skip(n),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Snapshot() Query {
+	return promisedQuery{
+		Query:   q.Query.Snapshot(),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Sort(fields ...string) Query {
+	return promisedQuery{
+		Query:   q.Query.Sort(fields...),
+		promise: q.promise,
+	}
+}
+
+func (q promisedQuery) Tail(timeout time.Duration) Iter {
+	return promisedIter{
+		Iter:    q.Query.Tail(timeout),
+		promise: q.promise,
+	}
+}
+
+func (q rejectedQuery) All(result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) Apply(change mgo.Change, result interface{}) (*mgo.ChangeInfo, error) {
+	return nil, breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) Batch(n int) Query {
+	return q
+}
+
+func (q rejectedQuery) Collation(collation *mgo.Collation) Query {
+	return q
+}
+
+func (q rejectedQuery) Comment(comment string) Query {
+	return q
+}
+
+func (q rejectedQuery) Count() (int, error) {
+	return 0, breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) Distinct(key string, result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) Explain(result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) For(result interface{}, f func() error) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) Hint(indexKey ...string) Query {
+	return q
+}
+
+func (q rejectedQuery) Iter() Iter {
+	return rejectedIter{}
+}
+
+func (q rejectedQuery) Limit(n int) Query {
+	return q
+}
+
+func (q rejectedQuery) LogReplay() Query {
+	return q
+}
+
+func (q rejectedQuery) MapReduce(job *mgo.MapReduce, result interface{}) (*mgo.MapReduceInfo, error) {
+	return nil, breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) One(result interface{}) error {
+	return breaker.ErrServiceUnavailable
+}
+
+func (q rejectedQuery) Prefetch(p float64) Query {
+	return q
+}
+
+func (q rejectedQuery) Select(selector interface{}) Query {
+	return q
+}
+
+func (q rejectedQuery) SetMaxScan(n int) Query {
+	return q
+}
+
+func (q rejectedQuery) SetMaxTime(d time.Duration) Query {
+	return q
+}
+
+func (q rejectedQuery) Skip(n int) Query {
+	return q
+}
+
+func (q rejectedQuery) Snapshot() Query {
+	return q
+}
+
+func (q rejectedQuery) Sort(fields ...string) Query {
+	return q
+}
+
+func (q rejectedQuery) Tail(timeout time.Duration) Iter {
+	return rejectedIter{}
+}

core/stores/mongo/query_test.go

@@ -0,0 +1,121 @@
+package mongo
+
+import (
+	"testing"
+
+	"zero/core/breaker"
+
+	"github.com/globalsign/mgo"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_rejectedQuery_All(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedQuery).All(nil))
+}
+
+func Test_rejectedQuery_Apply(t *testing.T) {
+	info, err := new(rejectedQuery).Apply(mgo.Change{}, nil)
+	assert.Equal(t, breaker.ErrServiceUnavailable, err)
+	assert.Nil(t, info)
+}
+
+func Test_rejectedQuery_Batch(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Batch(1))
+}
+
+func Test_rejectedQuery_Collation(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Collation(nil))
+}
+
+func Test_rejectedQuery_Comment(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Comment(""))
+}
+
+func Test_rejectedQuery_Count(t *testing.T) {
+	n, err := new(rejectedQuery).Count()
+	assert.Equal(t, breaker.ErrServiceUnavailable, err)
+	assert.Equal(t, 0, n)
+}
+
+func Test_rejectedQuery_Distinct(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedQuery).Distinct("", nil))
+}
+
+func Test_rejectedQuery_Explain(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedQuery).Explain(nil))
+}
+
+func Test_rejectedQuery_For(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedQuery).For(nil, nil))
+}
+
+func Test_rejectedQuery_Hint(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Hint())
+}
+
+func Test_rejectedQuery_Iter(t *testing.T) {
+	assert.EqualValues(t, rejectedIter{}, new(rejectedQuery).Iter())
+}
+
+func Test_rejectedQuery_Limit(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Limit(1))
+}
+
+func Test_rejectedQuery_LogReplay(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.LogReplay())
+}
+
+func Test_rejectedQuery_MapReduce(t *testing.T) {
+	info, err := new(rejectedQuery).MapReduce(nil, nil)
+	assert.Equal(t, breaker.ErrServiceUnavailable, err)
+	assert.Nil(t, info)
+}
+
+func Test_rejectedQuery_One(t *testing.T) {
+	assert.Equal(t, breaker.ErrServiceUnavailable, new(rejectedQuery).One(nil))
+}
+
+func Test_rejectedQuery_Prefetch(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Prefetch(1))
+}
+
+func Test_rejectedQuery_Select(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Select(nil))
+}
+
+func Test_rejectedQuery_SetMaxScan(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.SetMaxScan(0))
+}
+
+func Test_rejectedQuery_SetMaxTime(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.SetMaxTime(0))
+}
+
+func Test_rejectedQuery_Skip(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Skip(0))
+}
+
+func Test_rejectedQuery_Snapshot(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Snapshot())
+}
+
+func Test_rejectedQuery_Sort(t *testing.T) {
+	var q rejectedQuery
+	assert.Equal(t, q, q.Sort())
+}
+
+func Test_rejectedQuery_Tail(t *testing.T) {
+	assert.EqualValues(t, rejectedIter{}, new(rejectedQuery).Tail(0))
+}

core/stores/mongo/sessionmanager.go

@@ -0,0 +1,73 @@
+package mongo
+
+import (
+	"io"
+	"time"
+
+	"zero/core/logx"
+	"zero/core/syncx"
+
+	"github.com/globalsign/mgo"
+)
+
+const (
+	defaultConcurrency = 50
+	defaultTimeout     = time.Second
+)
+
+var sessionManager = syncx.NewResourceManager()
+
+type concurrentSession struct {
+	*mgo.Session
+	limit syncx.TimeoutLimit
+}
+
+func (cs *concurrentSession) Close() error {
+	cs.Session.Close()
+	return nil
+}
+
+func getConcurrentSession(url string) (*concurrentSession, error) {
+	val, err := sessionManager.GetResource(url, func() (io.Closer, error) {
+		mgoSession, err := mgo.Dial(url)
+		if err != nil {
+			return nil, err
+		}
+
+		concurrentSess := &concurrentSession{
+			Session: mgoSession,
+			limit:   syncx.NewTimeoutLimit(defaultConcurrency),
+		}
+
+		return concurrentSess, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return val.(*concurrentSession), nil
+}
+
+func (cs *concurrentSession) putSession(session *mgo.Session) {
+	if err := cs.limit.Return(); err != nil {
+		logx.Error(err)
+	}
+
+	// anyway, we need to close the session
+	session.Close()
+}
+
+func (cs *concurrentSession) takeSession(opts ...Option) (*mgo.Session, error) {
+	o := &options{
+		timeout: defaultTimeout,
+	}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	if err := cs.limit.Borrow(o.timeout); err != nil {
+		return nil, err
+	} else {
+		return cs.Copy(), nil
+	}
+}

core/stores/mongo/util.go

@@ -0,0 +1,9 @@
+package mongo
+
+import "strings"
+
+const mongoAddrSep = ","
+
+func FormatAddr(hosts []string) string {
+	return strings.Join(hosts, mongoAddrSep)
+}

core/stores/mongo/utils_test.go

@@ -0,0 +1,35 @@
+package mongo
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFormatAddrs(t *testing.T) {
+	tests := []struct {
+		addrs  []string
+		expect string
+	}{
+		{
+			addrs:  []string{"a", "b"},
+			expect: "a,b",
+		},
+		{
+			addrs:  []string{"a", "b", "c"},
+			expect: "a,b,c",
+		},
+		{
+			addrs:  []string{},
+			expect: "",
+		},
+		{
+			addrs:  nil,
+			expect: "",
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.expect, FormatAddr(test.addrs))
+	}
+}