initial import

2020-07-26 17:09:05 +08:00
commit 7e3a369a8f
647 changed files with 54754 additions and 0 deletions
--- a/example/load/main.go
+++ b/example/load/main.go
@@ -0,0 +1,149 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"math/rand"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"zero/core/collection"
+	"zero/core/executors"
+	"zero/core/lang"
+	"zero/core/syncx"
+
+	"gopkg.in/cheggaaa/pb.v1"
+)
+
+const (
+	beta     = 0.9
+	total    = 400
+	interval = time.Second
+	factor   = 5
+)
+
+var (
+	seconds             = flag.Int("d", 400, "duration to go")
+	flying              uint64
+	avgFlyingAggressive float64
+	aggressiveLock      syncx.SpinLock
+	avgFlyingLazy       float64
+	lazyLock            syncx.SpinLock
+	avgFlyingBoth       float64
+	bothLock            syncx.SpinLock
+	lessWriter          *executors.LessExecutor
+	passCounter         = collection.NewRollingWindow(50, time.Millisecond*100)
+	rtCounter           = collection.NewRollingWindow(50, time.Millisecond*100)
+	index               int32
+)
+
+func main() {
+	flag.Parse()
+
+	// only log 100 records
+	lessWriter = executors.NewLessExecutor(interval * total / 100)
+
+	fp, err := os.Create("result.csv")
+	lang.Must(err)
+	defer fp.Close()
+	fmt.Fprintln(fp, "second,maxFlight,flying,agressiveAvgFlying,lazyAvgFlying,bothAvgFlying")
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	bar := pb.New(*seconds * 2).Start()
+	var waitGroup sync.WaitGroup
+	batchRequests := func(i int) {
+		<-ticker.C
+		requests := (i + 1) * factor
+		func() {
+			it := time.NewTicker(interval / time.Duration(requests))
+			defer it.Stop()
+			for j := 0; j < requests; j++ {
+				<-it.C
+				waitGroup.Add(1)
+				go func() {
+					issueRequest(fp, atomic.AddInt32(&index, 1))
+					waitGroup.Done()
+				}()
+			}
+			bar.Increment()
+		}()
+	}
+	for i := 0; i < *seconds; i++ {
+		batchRequests(i)
+	}
+	for i := *seconds; i > 0; i-- {
+		batchRequests(i)
+	}
+	bar.Finish()
+	waitGroup.Wait()
+}
+
+func issueRequest(writer io.Writer, idx int32) {
+	v := atomic.AddUint64(&flying, 1)
+	aggressiveLock.Lock()
+	af := avgFlyingAggressive*beta + float64(v)*(1-beta)
+	avgFlyingAggressive = af
+	aggressiveLock.Unlock()
+	bothLock.Lock()
+	bf := avgFlyingBoth*beta + float64(v)*(1-beta)
+	avgFlyingBoth = bf
+	bothLock.Unlock()
+	duration := time.Millisecond * time.Duration(rand.Int63n(10)+1)
+	job(duration)
+	passCounter.Add(1)
+	rtCounter.Add(float64(duration) / float64(time.Millisecond))
+	v1 := atomic.AddUint64(&flying, ^uint64(0))
+	lazyLock.Lock()
+	lf := avgFlyingLazy*beta + float64(v1)*(1-beta)
+	avgFlyingLazy = lf
+	lazyLock.Unlock()
+	bothLock.Lock()
+	bf = avgFlyingBoth*beta + float64(v1)*(1-beta)
+	avgFlyingBoth = bf
+	bothLock.Unlock()
+	lessWriter.DoOrDiscard(func() {
+		fmt.Fprintf(writer, "%d,%d,%d,%.2f,%.2f,%.2f\n", idx, maxFlight(), v, af, lf, bf)
+	})
+}
+
+func job(duration time.Duration) {
+	time.Sleep(duration)
+}
+
+func maxFlight() int64 {
+	return int64(math.Max(1, float64(maxPass()*10)*(minRt()/1e3)))
+}
+
+func maxPass() int64 {
+	var result float64 = 1
+
+	passCounter.Reduce(func(b *collection.Bucket) {
+		if b.Sum > result {
+			result = b.Sum
+		}
+	})
+
+	return int64(result)
+}
+
+func minRt() float64 {
+	var result float64 = 1000
+
+	rtCounter.Reduce(func(b *collection.Bucket) {
+		if b.Count <= 0 {
+			return
+		}
+
+		avg := math.Round(b.Sum / float64(b.Count))
+		if avg < result {
+			result = avg
+		}
+	})
+
+	return result
+}
--- a/example/load/plot.py
+++ b/example/load/plot.py
@@ -0,0 +1,14 @@
+import click
+import pandas as pd
+import matplotlib.pyplot as plt
+
+@click.command()
+@click.option("--csv", default="result.csv")
+def main(csv):
+    df = pd.read_csv(csv, index_col="second")
+    df.drop(["agressiveAvgFlying", "bothAvgFlying"], axis=1, inplace=True)
+    df.plot()
+    plt.show()
+
+if __name__ == "__main__":
+    main()
--- a/example/load/simulate/client/main.go
+++ b/example/load/simulate/client/main.go
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sync/atomic"
+	"time"
+
+	"zero/core/fx"
+	"zero/core/lang"
+)
+
+var (
+	errServiceUnavailable = errors.New("service unavailable")
+	total                 int64
+	pass                  int64
+	fail                  int64
+	drop                  int64
+	seconds               int64 = 1
+)
+
+func main() {
+	flag.Parse()
+
+	fp, err := os.Create("result.csv")
+	lang.Must(err)
+	defer fp.Close()
+	fmt.Fprintln(fp, "seconds,total,pass,fail,drop")
+
+	go func() {
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for range ticker.C {
+			reset(fp)
+		}
+	}()
+
+	for i := 0; ; i++ {
+		it := time.NewTicker(time.Second / time.Duration(atomic.LoadInt64(&seconds)))
+		func() {
+			for j := 0; j < int(seconds); j++ {
+				<-it.C
+				go issueRequest()
+			}
+		}()
+		it.Stop()
+
+		cur := atomic.AddInt64(&seconds, 1)
+		fmt.Println(cur)
+	}
+}
+
+func issueRequest() {
+	atomic.AddInt64(&total, 1)
+	err := fx.DoWithTimeout(func() error {
+		return job()
+	}, time.Second)
+	switch err {
+	case nil:
+		atomic.AddInt64(&pass, 1)
+	case errServiceUnavailable:
+		atomic.AddInt64(&drop, 1)
+	default:
+		atomic.AddInt64(&fail, 1)
+	}
+}
+
+func job() error {
+	resp, err := http.Get("http://localhost:3333/")
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil
+	default:
+		return errServiceUnavailable
+	}
+}
+
+func reset(writer io.Writer) {
+	fmt.Fprintf(writer, "%d,%d,%d,%d,%d\n",
+		atomic.LoadInt64(&seconds),
+		atomic.SwapInt64(&total, 0),
+		atomic.SwapInt64(&pass, 0),
+		atomic.SwapInt64(&fail, 0),
+		atomic.SwapInt64(&drop, 0),
+	)
+}
--- a/example/load/simulate/client/plot.py
+++ b/example/load/simulate/client/plot.py
@@ -0,0 +1,13 @@
+import click
+import pandas as pd
+import matplotlib.pyplot as plt
+
+@click.command()
+@click.option("--csv", default="result.csv")
+def main(csv):
+    df = pd.read_csv(csv, index_col="seconds")
+    df.plot()
+    plt.show()
+
+if __name__ == "__main__":
+    main()
--- a/example/load/simulate/cpu/Dockerfile
+++ b/example/load/simulate/cpu/Dockerfile
@@ -0,0 +1,26 @@
+FROM golang:alpine AS builder
+
+LABEL stage=gobuilder
+
+ENV CGO_ENABLED 0
+ENV GOOS linux
+ENV GOPROXY https://goproxy.cn,direct
+
+WORKDIR $GOPATH/src/zero
+COPY . .
+RUN go build -ldflags="-s -w" -o /app/main example/load/simulate/cpu/main.go
+
+
+FROM alpine
+
+RUN apk add --no-cache tzdata
+ENV TZ Asia/Shanghai
+
+RUN apk add git
+RUN go get github.com/vikyd/go-cpu-load
+
+RUN mkdir /app
+COPY --from=builder /app/main /app/main
+
+WORKDIR /app
+CMD ["/app/main"]
--- a/example/load/simulate/cpu/Makefile
+++ b/example/load/simulate/cpu/Makefile
@@ -0,0 +1,13 @@
+version := v1
+
+build:
+	cd $(GOPATH)/src/zero && docker build -t registry.cn-hangzhou.aliyuncs.com/xapp/shedding:$(version) . -f example/load/simulate/cpu/Dockerfile
+
+push: build
+	docker push registry.cn-hangzhou.aliyuncs.com/xapp/shedding:$(version)
+
+deploy: push
+	kubectl apply -f shedding.yaml
+
+clean:
+	kubectl delete -f shedding.yaml
--- a/example/load/simulate/cpu/cpu-accuracy.md
+++ b/example/load/simulate/cpu/cpu-accuracy.md
@@ -0,0 +1,28 @@
+# cpu监控准确度测试
+
+1. 启动测试pod
+
+   `make deploy`
+
+2. 通过`kubectl get po -n adhoc`确认`sheeding` pod已经成功运行，通过如下命令进入pod
+
+   `kubectl exec -it -n adhoc shedding -- sh`
+
+3. 启动负载
+
+   `/app # go-cpu-load -p 50 -c 1`
+
+   默认`go-cpu-load`是对每个core加上负载的，所以测试里指定了`1000m`，等同于1 core，我们指定`-c 1`让测试更具有可读性
+
+   `-p`可以多换几个值测试
+
+4. 验证测试准确性
+
+   `kubectl logs -f -n adhoc shedding`
+
+   可以看到日志中的`CPU`报告，`1000m`表示`100%`，如果看到`500m`则表示`50%`，每分钟输出一次
+
+   `watch -n 5 kubectl top pod -n adhoc`
+
+   可以看到`kubectl`报告的`CPU`使用率，两者进行对比，即可知道是否准确
+
--- a/example/load/simulate/cpu/main.go
+++ b/example/load/simulate/cpu/main.go
@@ -0,0 +1,7 @@
+package main
+
+import _ "zero/core/stat"
+
+func main() {
+	select {}
+}
--- a/example/load/simulate/cpu/shedding.yaml
+++ b/example/load/simulate/cpu/shedding.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: shedding
+  namespace: adhoc
+spec:
+  containers:
+    - name: shedding
+      image: registry-vpc.cn-hangzhou.aliyuncs.com/xapp/shedding:v1
+      imagePullPolicy: Always
+      resources:
+        requests:
+          cpu: 1000m
+        limits:
+          cpu: 1000m
+  imagePullSecrets:
+    - name: aliyun
--- a/example/load/simulate/server/server.go
+++ b/example/load/simulate/server/server.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"runtime"
+	"time"
+
+	"zero/core/fx"
+	"zero/core/logx"
+	"zero/core/service"
+	"zero/core/stat"
+	"zero/ngin"
+)
+
+const duration = time.Millisecond
+
+func main() {
+	go func() {
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for range ticker.C {
+			fmt.Printf("cpu: %d\n", stat.CpuUsage())
+		}
+	}()
+
+	logx.Disable()
+	engine := ngin.MustNewEngine(ngin.NgConf{
+		ServiceConf: service.ServiceConf{
+			Log: logx.LogConf{
+				Mode: "console",
+			},
+		},
+		Host:         "0.0.0.0",
+		Port:         3333,
+		CpuThreshold: 800,
+	})
+	defer engine.Stop()
+	engine.AddRoute(ngin.Route{
+		Method: http.MethodGet,
+		Path:   "/",
+		Handler: func(w http.ResponseWriter, r *http.Request) {
+			if err := fx.DoWithTimeout(func() error {
+				job(duration)
+				return nil
+			}, time.Millisecond*100); err != nil {
+				w.WriteHeader(http.StatusServiceUnavailable)
+			}
+		},
+	})
+	engine.Start()
+}
+
+func job(duration time.Duration) {
+	done := make(chan int)
+
+	for i := 0; i < runtime.NumCPU(); i++ {
+		go func() {
+			for {
+				select {
+				case <-done:
+					return
+				default:
+				}
+			}
+		}()
+	}
+
+	time.Sleep(duration)
+	close(done)
+}