initial import

stash/config/config.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"time"
+
+	"zero/kq"
+)
+
+type (
+	Condition struct {
+		Key   string
+		Value string
+		Type  string `json:",default=match,options=match|contains"`
+		Op    string `json:",default=and,options=and|or"`
+	}
+
+	ElasticSearchConf struct {
+		Hosts            []string
+		DailyIndexPrefix string
+		TimeZone         string `json:",optional"`
+		MaxChunkBytes    int    `json:",default=1048576"`
+		Compress         bool   `json:",default=false"`
+	}
+
+	Filter struct {
+		Action     string      `json:",options=drop|remove_field"`
+		Conditions []Condition `json:",optional"`
+		Fields     []string    `json:",optional"`
+	}
+
+	Config struct {
+		Input struct {
+			Kafka kq.KqConf
+		}
+		Filters []Filter
+		Output  struct {
+			ElasticSearch ElasticSearchConf
+		}
+		GracePeriod time.Duration `json:",default=10s"`
+	}
+)

stash/es/index.go

@@ -0,0 +1,82 @@
+package es
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"zero/core/fx"
+	"zero/core/logx"
+	"zero/core/syncx"
+
+	"github.com/olivere/elastic"
+)
+
+const sharedCallsKey = "ensureIndex"
+
+type (
+	IndexFormat func(time.Time) string
+	IndexFunc   func() string
+
+	Index struct {
+		client      *elastic.Client
+		indexFormat IndexFormat
+		index       string
+		lock        sync.RWMutex
+		sharedCalls syncx.SharedCalls
+	}
+)
+
+func NewIndex(client *elastic.Client, indexFormat IndexFormat) *Index {
+	return &Index{
+		client:      client,
+		indexFormat: indexFormat,
+		sharedCalls: syncx.NewSharedCalls(),
+	}
+}
+
+func (idx *Index) GetIndex(t time.Time) string {
+	index := idx.indexFormat(t)
+	if err := idx.ensureIndex(index); err != nil {
+		logx.Error(err)
+	}
+	return index
+}
+
+func (idx *Index) ensureIndex(index string) error {
+	idx.lock.RLock()
+	if index == idx.index {
+		idx.lock.RUnlock()
+		return nil
+	}
+	idx.lock.RUnlock()
+
+	_, err := idx.sharedCalls.Do(sharedCallsKey, func() (i interface{}, err error) {
+		idx.lock.Lock()
+		defer idx.lock.Unlock()
+
+		existsService := elastic.NewIndicesExistsService(idx.client)
+		existsService.Index([]string{index})
+		exist, err := existsService.Do(context.Background())
+		if err != nil {
+			return nil, err
+		}
+		if exist {
+			idx.index = index
+			return nil, nil
+		}
+
+		createService := idx.client.CreateIndex(index)
+		if err := fx.DoWithRetries(func() error {
+			// is it necessary to check the result?
+			_, err := createService.Do(context.Background())
+			return err
+		}); err != nil {
+			return nil, err
+		}
+
+		idx.index = index
+		return nil, nil
+	})
+	return err
+}

stash/es/writer.go

@@ -0,0 +1,65 @@
+package es
+
+import (
+	"context"
+	"time"
+
+	"zero/core/executors"
+	"zero/core/logx"
+	"zero/stash/config"
+
+	"github.com/olivere/elastic"
+)
+
+const docType = "doc"
+
+type (
+	Writer struct {
+		client   *elastic.Client
+		indexer  *Index
+		inserter *executors.ChunkExecutor
+	}
+
+	valueWithTime struct {
+		t   time.Time
+		val string
+	}
+)
+
+func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) {
+	client, err := elastic.NewClient(
+		elastic.SetSniff(false),
+		elastic.SetURL(c.Hosts...),
+		elastic.SetGzip(c.Compress),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	writer := Writer{
+		client:  client,
+		indexer: indexer,
+	}
+	writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes))
+	return &writer, nil
+}
+
+func (w *Writer) Write(t time.Time, val string) error {
+	return w.inserter.Add(valueWithTime{
+		t:   t,
+		val: val,
+	}, len(val))
+}
+
+func (w *Writer) execute(vals []interface{}) {
+	var bulk = w.client.Bulk()
+	for _, val := range vals {
+		pair := val.(valueWithTime)
+		req := elastic.NewBulkIndexRequest().Index(w.indexer.GetIndex(pair.t)).Type(docType).Doc(pair.val)
+		bulk.Add(req)
+	}
+	_, err := bulk.Do(context.Background())
+	if err != nil {
+		logx.Error(err)
+	}
+}

stash/etc/config.json

@@ -0,0 +1,75 @@
+{
+    "Input": {
+        "Kafka": {
+            "Name": "easystash",
+            "Brokers": [
+                "172.16.186.156:19092",
+                "172.16.186.157:19092",
+                "172.16.186.158:19092",
+                "172.16.186.159:19092",
+                "172.16.186.160:19092",
+                "172.16.186.161:19092"
+            ],
+            "Topic": "k8slog",
+            "Group": "pro",
+            "NumProducers": 16,
+            "MetricsUrl": "http://localhost:2222/add"
+        }
+    },
+    "Filters": [
+        {
+            "Action": "drop",
+            "Conditions": [
+                {
+                    "Key": "k8s_container_name",
+                    "Value": "-rpc",
+                    "Type": "contains"
+                },
+                {
+                    "Key": "level",
+                    "Value": "info",
+                    "Type": "match",
+                    "Op": "and"
+                }
+            ]
+        },
+        {
+            "Action": "remove_field",
+            "Fields": [
+                "message",
+                "_source",
+                "_type",
+                "_score",
+                "_id",
+                "@version",
+                "topic",
+                "index",
+                "beat",
+                "docker_container",
+                "offset",
+                "prospector",
+                "source",
+                "stream"
+            ]
+        }
+    ],
+    "Output": {
+        "ElasticSearch": {
+            "Hosts": [
+                "172.16.141.14:9200",
+                "172.16.141.15:9200",
+                "172.16.141.16:9200",
+                "172.16.141.17:9200",
+                "172.16.140.195:9200",
+                "172.16.140.196:9200",
+                "172.16.140.197:9200",
+                "172.16.140.198:9200",
+                "172.16.140.199:9200",
+                "172.16.140.200:9200",
+                "172.16.140.201:9200",
+                "172.16.140.202:9200"
+            ],
+            "DailyIndexPrefix": "k8s_pro-"
+        }
+    }
+}

stash/filter/filters.go

@@ -0,0 +1,104 @@
+package filter
+
+import (
+	"strings"
+
+	"zero/stash/config"
+
+	"github.com/globalsign/mgo/bson"
+)
+
+const (
+	filterDrop         = "drop"
+	filterRemoveFields = "remove_field"
+	opAnd              = "and"
+	opOr               = "or"
+	typeContains       = "contains"
+	typeMatch          = "match"
+)
+
+type FilterFunc func(map[string]interface{}) map[string]interface{}
+
+func CreateFilters(c config.Config) []FilterFunc {
+	var filters []FilterFunc
+
+	for _, f := range c.Filters {
+		switch f.Action {
+		case filterDrop:
+			filters = append(filters, DropFilter(f.Conditions))
+		case filterRemoveFields:
+			filters = append(filters, RemoveFieldFilter(f.Fields))
+		}
+	}
+
+	return filters
+}
+
+func DropFilter(conds []config.Condition) FilterFunc {
+	return func(m map[string]interface{}) map[string]interface{} {
+		var qualify bool
+		for _, cond := range conds {
+			var qualifyOnce bool
+			switch cond.Type {
+			case typeMatch:
+				qualifyOnce = cond.Value == m[cond.Key]
+			case typeContains:
+				if val, ok := m[cond.Key].(string); ok {
+					qualifyOnce = strings.Contains(val, cond.Value)
+				}
+			}
+
+			switch cond.Op {
+			case opAnd:
+				if !qualifyOnce {
+					return m
+				} else {
+					qualify = true
+				}
+			case opOr:
+				if qualifyOnce {
+					qualify = true
+				}
+			}
+		}
+
+		if qualify {
+			return nil
+		} else {
+			return m
+		}
+	}
+}
+
+func RemoveFieldFilter(fields []string) FilterFunc {
+	return func(m map[string]interface{}) map[string]interface{} {
+		for _, field := range fields {
+			delete(m, field)
+		}
+		return m
+	}
+}
+
+func AddUriFieldFilter(inField, outFirld string) FilterFunc {
+	return func(m map[string]interface{}) map[string]interface{} {
+		if val, ok := m[inField].(string); ok {
+			var datas []string
+			idx := strings.Index(val, "?")
+			if idx < 0 {
+				datas = strings.Split(val, "/")
+			} else {
+				datas = strings.Split(val[:idx], "/")
+			}
+
+			for i, data := range datas {
+				if bson.IsObjectIdHex(data) {
+					datas[i] = "*"
+				}
+			}
+
+			m[outFirld] = strings.Join(datas, "/")
+		}
+
+		return m
+	}
+}

stash/handler/handler.go

@@ -0,0 +1,64 @@
+package handler
+
+import (
+	"time"
+
+	"zero/stash/es"
+	"zero/stash/filter"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+const (
+	timestampFormat = "2006-01-02T15:04:05.000Z"
+	timestampKey    = "@timestamp"
+)
+
+type MessageHandler struct {
+	writer  *es.Writer
+	filters []filter.FilterFunc
+}
+
+func NewHandler(writer *es.Writer) *MessageHandler {
+	return &MessageHandler{
+		writer: writer,
+	}
+}
+
+func (mh *MessageHandler) AddFilters(filters ...filter.FilterFunc) {
+	for _, f := range filters {
+		mh.filters = append(mh.filters, f)
+	}
+}
+
+func (mh *MessageHandler) Consume(_, val string) error {
+	m := make(map[string]interface{})
+	if err := jsoniter.Unmarshal([]byte(val), &m); err != nil {
+		return err
+	}
+
+	for _, proc := range mh.filters {
+		if m = proc(m); m == nil {
+			return nil
+		}
+	}
+
+	bs, err := jsoniter.Marshal(m)
+	if err != nil {
+		return err
+	}
+
+	return mh.writer.Write(mh.getTime(m), string(bs))
+}
+
+func (mh *MessageHandler) getTime(m map[string]interface{}) time.Time {
+	if ti, ok := m[timestampKey]; ok {
+		if ts, ok := ti.(string); ok {
+			if t, err := time.Parse(timestampFormat, ts); err == nil {
+				return t
+			}
+		}
+	}
+
+	return time.Now()
+}

stash/stash.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"flag"
+	"time"
+
+	"zero/core/conf"
+	"zero/core/lang"
+	"zero/core/proc"
+	"zero/kq"
+	"zero/stash/config"
+	"zero/stash/es"
+	"zero/stash/filter"
+	"zero/stash/handler"
+
+	"github.com/olivere/elastic"
+)
+
+const dateFormat = "2006.01.02"
+
+var configFile = flag.String("f", "etc/config.json", "Specify the config file")
+
+func main() {
+	flag.Parse()
+
+	var c config.Config
+	conf.MustLoad(*configFile, &c)
+	proc.SetTimeoutToForceQuit(c.GracePeriod)
+
+	client, err := elastic.NewClient(
+		elastic.SetSniff(false),
+		elastic.SetURL(c.Output.ElasticSearch.Hosts...),
+	)
+	lang.Must(err)
+
+	indexFormat := c.Output.ElasticSearch.DailyIndexPrefix + dateFormat
+	var loc *time.Location
+	if len(c.Output.ElasticSearch.TimeZone) > 0 {
+		loc, err = time.LoadLocation(c.Output.ElasticSearch.TimeZone)
+		lang.Must(err)
+	} else {
+		loc = time.Local
+	}
+	indexer := es.NewIndex(client, func(t time.Time) string {
+		return t.In(loc).Format(indexFormat)
+	})
+
+	filters := filter.CreateFilters(c)
+	writer, err := es.NewWriter(c.Output.ElasticSearch, indexer)
+	lang.Must(err)
+
+	handle := handler.NewHandler(writer)
+	handle.AddFilters(filters...)
+	handle.AddFilters(filter.AddUriFieldFilter("url", "uri"))
+	q := kq.MustNewQueue(c.Input.Kafka, handle)
+	q.Start()
+}