initial import

core/redisqueue/conf.go

@@ -0,0 +1,19 @@
+package redisqueue
+
+import (
+	"zero/core/queue"
+	"zero/core/stores/redis"
+)
+
+type RedisKeyConf struct {
+	redis.RedisConf
+	Key string `json:",optional"`
+}
+
+func (rkc RedisKeyConf) NewProducer(opts ...ProducerOption) (queue.Producer, error) {
+	return newProducer(rkc.NewRedis(), rkc.Key, opts...)
+}
+
+func (rkc RedisKeyConf) NewPusher(opts ...PusherOption) queue.QueuePusher {
+	return NewPusher(rkc.NewRedis(), rkc.Key, opts...)
+}

core/redisqueue/message.go

@@ -0,0 +1,6 @@
+package redisqueue
+
+type TimedMessage struct {
+	Time    int64  `json:"time"`
+	Payload string `json:"payload"`
+}

core/redisqueue/redisqueue_test.go

@@ -0,0 +1,82 @@
+package redisqueue
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	"zero/core/logx"
+	"zero/core/queue"
+	"zero/core/stores/redis"
+
+	"github.com/alicebob/miniredis"
+	"github.com/stretchr/testify/assert"
+)
+
+func init() {
+	logx.Disable()
+}
+
+func TestRedisQueue(t *testing.T) {
+	const (
+		total = 1000
+		key   = "queue"
+	)
+	r, err := miniredis.Run()
+	assert.Nil(t, err)
+
+	c := RedisKeyConf{
+		RedisConf: redis.RedisConf{
+			Host: r.Addr(),
+			Type: redis.NodeType,
+		},
+		Key: key,
+	}
+
+	pusher := NewPusher(c.NewRedis(), key, WithTime())
+	assert.True(t, len(pusher.Name()) > 0)
+	for i := 0; i < total; i++ {
+		err := pusher.Push(strconv.Itoa(i))
+		assert.Nil(t, err)
+	}
+
+	consumer := new(mockedConsumer)
+	consumer.wait.Add(total)
+	q := queue.NewQueue(func() (queue.Producer, error) {
+		return c.NewProducer(TimeSensitive(5))
+	}, func() (queue.Consumer, error) {
+		return consumer, nil
+	})
+	q.SetNumProducer(1)
+	q.SetNumConsumer(1)
+	go func() {
+		q.Start()
+	}()
+	consumer.wait.Wait()
+	q.Stop()
+
+	var expect int
+	for i := 0; i < total; i++ {
+		expect ^= i
+	}
+	assert.Equal(t, expect, consumer.xor)
+}
+
+type mockedConsumer struct {
+	wait sync.WaitGroup
+	xor  int
+}
+
+func (c *mockedConsumer) Consume(s string) error {
+	val, err := strconv.Atoi(s)
+	if err != nil {
+		return err
+	}
+
+	c.xor ^= val
+	c.wait.Done()
+	return nil
+}
+
+func (c *mockedConsumer) OnEvent(event interface{}) {
+}

core/redisqueue/redisqueueproducer.go

@@ -0,0 +1,166 @@
+package redisqueue
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"zero/core/jsonx"
+	"zero/core/logx"
+	"zero/core/queue"
+	"zero/core/stores/redis"
+)
+
+const (
+	logIntervalMillis  = 1000
+	retryRedisInterval = time.Second
+)
+
+type (
+	ProducerOption func(p queue.Producer) queue.Producer
+
+	RedisQueueProducer struct {
+		name      string
+		store     *redis.Redis
+		key       string
+		redisNode redis.ClosableNode
+		listeners []queue.ProduceListener
+	}
+)
+
+func NewProducerFactory(store *redis.Redis, key string, opts ...ProducerOption) queue.ProducerFactory {
+	return func() (queue.Producer, error) {
+		return newProducer(store, key, opts...)
+	}
+}
+
+func (p *RedisQueueProducer) AddListener(listener queue.ProduceListener) {
+	p.listeners = append(p.listeners, listener)
+}
+
+func (p *RedisQueueProducer) Name() string {
+	return p.name
+}
+
+func (p *RedisQueueProducer) Produce() (string, bool) {
+	lessLogger := logx.NewLessLogger(logIntervalMillis)
+
+	for {
+		value, ok, err := p.store.BlpopEx(p.redisNode, p.key)
+		if err == nil {
+			return value, ok
+		} else if err == redis.Nil {
+			// timed out without elements popped
+			continue
+		} else {
+			lessLogger.Errorf("Error on blpop: %v", err)
+			p.waitForRedisAvailable()
+		}
+	}
+}
+
+func newProducer(store *redis.Redis, key string, opts ...ProducerOption) (queue.Producer, error) {
+	redisNode, err := redis.CreateBlockingNode(store)
+	if err != nil {
+		return nil, err
+	}
+
+	var producer queue.Producer = &RedisQueueProducer{
+		name:      fmt.Sprintf("%s/%s/%s", store.Type, store.Addr, key),
+		store:     store,
+		key:       key,
+		redisNode: redisNode,
+	}
+
+	for _, opt := range opts {
+		producer = opt(producer)
+	}
+
+	return producer, nil
+}
+
+func (p *RedisQueueProducer) resetRedisConnection() error {
+	if p.redisNode != nil {
+		p.redisNode.Close()
+		p.redisNode = nil
+	}
+
+	redisNode, err := redis.CreateBlockingNode(p.store)
+	if err != nil {
+		return err
+	}
+
+	p.redisNode = redisNode
+	return nil
+}
+
+func (p *RedisQueueProducer) waitForRedisAvailable() {
+	var paused bool
+	var pauseOnce sync.Once
+
+	for {
+		if err := p.resetRedisConnection(); err != nil {
+			pauseOnce.Do(func() {
+				paused = true
+				for _, listener := range p.listeners {
+					listener.OnProducerPause()
+				}
+			})
+			logx.Errorf("Error occurred while connect to redis: %v", err)
+			time.Sleep(retryRedisInterval)
+		} else {
+			break
+		}
+	}
+
+	if paused {
+		for _, listener := range p.listeners {
+			listener.OnProducerResume()
+		}
+	}
+}
+
+func TimeSensitive(seconds int64) ProducerOption {
+	return func(p queue.Producer) queue.Producer {
+		if seconds > 0 {
+			return autoDropQueueProducer{
+				seconds:  seconds,
+				producer: p,
+			}
+		}
+
+		return p
+	}
+}
+
+type autoDropQueueProducer struct {
+	seconds  int64 // seconds before to drop
+	producer queue.Producer
+}
+
+func (p autoDropQueueProducer) AddListener(listener queue.ProduceListener) {
+	p.producer.AddListener(listener)
+}
+
+func (p autoDropQueueProducer) Produce() (string, bool) {
+	lessLogger := logx.NewLessLogger(logIntervalMillis)
+
+	for {
+		content, ok := p.producer.Produce()
+		if !ok {
+			return "", false
+		}
+
+		var timedMsg TimedMessage
+		if err := jsonx.UnmarshalFromString(content, &timedMsg); err != nil {
+			lessLogger.Errorf("invalid timedMessage: %s, error: %s", content, err.Error())
+			continue
+		}
+
+		if timedMsg.Time+p.seconds < time.Now().Unix() {
+			lessLogger.Errorf("expired timedMessage: %s", content)
+		}
+
+		return timedMsg.Payload, true
+	}
+}

core/redisqueue/redisqueuepusher.go

@@ -0,0 +1,78 @@
+package redisqueue
+
+import (
+	"fmt"
+	"time"
+
+	"zero/core/jsonx"
+	"zero/core/logx"
+	"zero/core/queue"
+	"zero/core/stores/redis"
+)
+
+type (
+	PusherOption func(p queue.QueuePusher) queue.QueuePusher
+
+	RedisQueuePusher struct {
+		name  string
+		store *redis.Redis
+		key   string
+	}
+)
+
+func NewPusher(store *redis.Redis, key string, opts ...PusherOption) queue.QueuePusher {
+	var pusher queue.QueuePusher = &RedisQueuePusher{
+		name:  fmt.Sprintf("%s/%s/%s", store.Type, store.Addr, key),
+		store: store,
+		key:   key,
+	}
+
+	for _, opt := range opts {
+		pusher = opt(pusher)
+	}
+
+	return pusher
+}
+
+func (saver *RedisQueuePusher) Name() string {
+	return saver.name
+}
+
+func (saver *RedisQueuePusher) Push(message string) error {
+	_, err := saver.store.Rpush(saver.key, message)
+	if nil != err {
+		return err
+	}
+
+	logx.Infof("<= %s", message)
+	return nil
+}
+
+func WithTime() PusherOption {
+	return func(p queue.QueuePusher) queue.QueuePusher {
+		return timedQueuePusher{
+			pusher: p,
+		}
+	}
+}
+
+type timedQueuePusher struct {
+	pusher queue.QueuePusher
+}
+
+func (p timedQueuePusher) Name() string {
+	return p.pusher.Name()
+}
+
+func (p timedQueuePusher) Push(message string) error {
+	tm := TimedMessage{
+		Time:    time.Now().Unix(),
+		Payload: message,
+	}
+
+	if content, err := jsonx.Marshal(tm); err != nil {
+		return err
+	} else {
+		return p.pusher.Push(string(content))
+	}
+}